wassname/FanFicFare
1
0
Fork 0
mirror of https://github.com/wassname/FanFicFare.git synced 2026-09-10 11:40:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add additional features to/for forums(SB&SV) adapters.

Browse Source
This commit is contained in:
Jim Miller
2015-06-25 13:18:47 -05:00
parent 78f1760857
commit ea2f64a7fb
5 changed files with 269 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
calibre-plugin/plugin-defaults.ini
+100 -3
View File
@@ -330,9 +330,10 @@ sort_ships:false
## User-agent
user_agent:FFF/2.X
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
## At the time of writing, eFiction Base adapters allow downloading
## the whole story in bulk using the 'Print' feature. If 'bulk_load'
## is set to 'true', both metadata and chapters can be loaded in one
## step
bulk_load:true
## Each output format has a section that overrides [defaults]
@@ -1040,6 +1041,102 @@ extra_valid_entries:size
# don't show twitter icon.
cover_exclusion_regexp:/res/css/bir.png
[forums.sufficientvelocity.com]
cover_exclusion_regexp:/clear.png
add_to_extratags:ForumFic
strip_chapter_numbers:false
# true, false, threadmarksonly
add_chapter_dates:false
add_to_extra_valid_entries:,titletags
# '.NOREPL' tells the system to *not* apply title's
# in/exclude/replace_metadata -- Only works on include_in_ lines.
include_in_titletags:title.NOREPL
## might want to do this, maybe not. Will often include category, but
## also often include non-category stuff.
# include_in_category:titletags
include_metadata_pre:
# only keep titletags with ( or [ in.
titletags=~[\[\(]
replace_metadata:
# remove 'Thread' and the next word, usually "Thread 2", "Thread
# four", "Thread iv", etc
title=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
# remove anything outside () or []
titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
# remove () []
titletags=>[\(\)\[\]]=>
# change (spaces)slash(spaces) to comma
titletags=> */ *=>,
titletags=> x =>,
# remove [] or () blocks and leading/trailing spaces
title=> *[\(\[]([^\]\)]+)[\)\]] *=>
extra_titlepage_entries: titletags
## '.SPLIT' teels the system to split by ','
add_to_include_subject_tags:,titletags.SPLIT
datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
description_limit:1500
[forums.spacebattles.com]
cover_exclusion_regexp:/clear.png
add_to_extratags:ForumFic
# true, false, threadmarksonly
add_chapter_dates:false
strip_chapter_numbers:false
add_to_extra_valid_entries:,titletags
# '.NOREPL' tells the system to *not* apply title's
# in/exclude/replace_metadata -- Only works on include_in_ lines.
include_in_titletags:title.NOREPL
## might want to do this, maybe not. Will often include category, but
## also often include non-category stuff.
# include_in_category:titletags
include_metadata_pre:
# only keep titletags with ( or [ in.
titletags=~[\[\(]
replace_metadata:
# remove 'Thread' and the next word, usually "Thread 2", "Thread
# four", "Thread iv", etc
title=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
# remove anything outside () or []
titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
# remove () []
titletags=>[\(\)\[\]]=>
# change (spaces)slash(spaces) to comma
titletags=> */ *=>,
titletags=> x =>,
# remove [] or () blocks and leading/trailing spaces
title=> *[\(\[]([^\]\)]+)[\)\]] *=>
extra_titlepage_entries: titletags
## '.SPLIT' teels the system to split by ','
add_to_include_subject_tags:,titletags.SPLIT
datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
description_limit:1500
[grangerenchanted.com]
## Some sites require login (or login for some rated stories) The
## program can prompt you, or you can save it in config. In
fanficfare/adapters/adapter_forumsspacebattlescom.py
+45 -17
View File
@@ -64,7 +64,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter):
# The date format will vary from site to site.
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
#self.dateformat = "%Y-%b-%d"
self.dateformat = "%b %d, %Y at %I:%M %p"
@staticmethod # must be @staticmethod, don't remove it.
def getSiteDomain():
@@ -93,13 +93,13 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter):
## Getting the chapter list and the meta data, plus 'is adult' checking.
def extractChapterUrlsAndMetadata(self):
url = self.url
logger.info("url: "+url)
useurl = self.url
logger.info("url: "+useurl)
try:
(data,opened) = self._fetchUrlOpened(url)
url = opened.geturl()
logger.info("use url: "+url)
(data,opened) = self._fetchUrlOpened(useurl)
useurl = opened.geturl()
logger.info("use useurl: "+useurl)
except urllib2.HTTPError, e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -114,13 +114,11 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter):
self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href'])
self.story.addToList('author',a.text)
self.story.addToList('genre','ForumFic')
h1 = soup.find('div',{'class':'titleBar'}).h1
self.story.setMetadata('title',stripHTML(h1))
if '#' in url:
anchorid = url.split('#')[1]
if '#' in useurl:
anchorid = useurl.split('#')[1]
soup = soup.find('li',id=anchorid)
else:
# try threadmarks if no '#' in , require at least 2.
@@ -129,17 +127,43 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter):
soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href']))
markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a')
if len(markas) > 1:
for (url,name) in [ (x['href'],stripHTML(x)) for x in markas ]:
for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]:
datestr=None
datetag = atag.find_next_sibling('div',{'class':'extra'}).find('span',{'class':'DateTime'})
if datetag:
datestr = datetag['title']
else:
datetag = atag.find_next_sibling('div',{'class':'extra'}).find('abbr',{'class':'DateTime'})
if datetag:
datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring'])
# Apr 24, 2015 at 4:39 AM
# May 1, 2015 at 5:47 AM
datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours.
date = makeDate(datestr, self.dateformat)
if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
self.story.setMetadata('datePublished', date)
if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'):
self.story.setMetadata('dateUpdated', date)
if self.getConfig('add_chapter_dates') in ['true','threadmarksonly']:
name = '%s %s'%(name,date)
self.chapterUrls.append((name,self.getURLPrefix()+'/'+url))
# Now go hunting for the 'chapter list'.
firstpost = soup.find('blockquote') # assume first posting contains TOC urls.
bq = soup.find('blockquote') # assume first posting contains TOC urls.
bq.name='div'
for iframe in bq.find_all('iframe'):
iframe.extract() # calibre book reader & editor don't like iframes to youtube.
self.setDescription(useurl,bq)
# otherwise, use first post links--include first post since that's
if not self.chapterUrls:
#logger.debug("len(firstpost):%s"%len(unicode(firstpost)))
self.chapterUrls.append(("First Post",self.url))
for (url,name) in [ (x['href'],stripHTML(x)) for x in firstpost.find_all('a') ]:
self.chapterUrls.append(("First Post",useurl))
for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]:
logger.debug("found chapurl:%s"%url)
if not url.startswith('http'):
url = self.getURLPrefix()+'/'+url
@@ -147,8 +171,12 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter):
if (url.startswith(self.getURLPrefix()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url):
# brute force way to deal with SB's http->https change when hardcoded http urls.
url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix())
logger.debug("used chapurl:%s"%url)
logger.debug("used chapurl:%s"%(url))
self.chapterUrls.append((name,url))
if url == useurl and 'First Post' == self.chapterUrls[0][0]:
# remove "First Post" if included in list.
logger.debug("delete dup chapter: %s %s"%self.chapterUrls[0])
del self.chapterUrls[0]
self.story.setMetadata('numChapters',len(self.chapterUrls))
fanficfare/adapters/base_adapter.py
+7
View File
@@ -497,6 +497,13 @@ class BaseSiteAdapter(Configurable):
def setDescription(self,url,svalue):
#print("\n\nsvalue:\n%s\n"%svalue)
strval = u"%s"%svalue # works for either soup or string
if self.hasConfig('description_limit'):
limit = int(self.getConfig('description_limit'))
if limit and len(strval) > limit:
svalue = strval[:limit]
#print(u"[[[[[\n\n%s\n\n]]]]]]]]"%svalue) # works for either soup or string
if self.getConfig('keep_summary_html'):
if isinstance(svalue,basestring):
# bs4/html5lib add html, header and body tags, which
fanficfare/defaults.ini
+100 -3
View File
@@ -327,9 +327,10 @@ sort_ships:false
## User-agent
user_agent:FFF/2.X
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
## At the time of writing, eFiction Base adapters allow downloading
## the whole story in bulk using the 'Print' feature. If 'bulk_load'
## is set to 'true', both metadata and chapters can be loaded in one
## step
bulk_load:true
## Each output format has a section that overrides [defaults]
@@ -1026,6 +1027,102 @@ extra_valid_entries:size
# don't show twitter icon.
cover_exclusion_regexp:/res/css/bir.png
[forums.sufficientvelocity.com]
cover_exclusion_regexp:/clear.png
add_to_extratags:ForumFic
strip_chapter_numbers:false
# true, false, threadmarksonly
add_chapter_dates:false
add_to_extra_valid_entries:,titletags
# '.NOREPL' tells the system to *not* apply title's
# in/exclude/replace_metadata -- Only works on include_in_ lines.
include_in_titletags:title.NOREPL
## might want to do this, maybe not. Will often include category, but
## also often include non-category stuff.
# include_in_category:titletags
include_metadata_pre:
# only keep titletags with ( or [ in.
titletags=~[\[\(]
replace_metadata:
# remove 'Thread' and the next word, usually "Thread 2", "Thread
# four", "Thread iv", etc
title=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
# remove anything outside () or []
titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
# remove () []
titletags=>[\(\)\[\]]=>
# change (spaces)slash(spaces) to comma
titletags=> */ *=>,
titletags=> x =>,
# remove [] or () blocks and leading/trailing spaces
title=> *[\(\[]([^\]\)]+)[\)\]] *=>
extra_titlepage_entries: titletags
## '.SPLIT' teels the system to split by ','
add_to_include_subject_tags:,titletags.SPLIT
datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
description_limit:1500
[forums.spacebattles.com]
cover_exclusion_regexp:/clear.png
add_to_extratags:ForumFic
# true, false, threadmarksonly
add_chapter_dates:false
strip_chapter_numbers:false
add_to_extra_valid_entries:,titletags
# '.NOREPL' tells the system to *not* apply title's
# in/exclude/replace_metadata -- Only works on include_in_ lines.
include_in_titletags:title.NOREPL
## might want to do this, maybe not. Will often include category, but
## also often include non-category stuff.
# include_in_category:titletags
include_metadata_pre:
# only keep titletags with ( or [ in.
titletags=~[\[\(]
replace_metadata:
# remove 'Thread' and the next word, usually "Thread 2", "Thread
# four", "Thread iv", etc
title=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
# remove anything outside () or []
titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
# remove () []
titletags=>[\(\)\[\]]=>
# change (spaces)slash(spaces) to comma
titletags=> */ *=>,
titletags=> x =>,
# remove [] or () blocks and leading/trailing spaces
title=> *[\(\[]([^\]\)]+)[\)\]] *=>
extra_titlepage_entries: titletags
## '.SPLIT' teels the system to split by ','
add_to_include_subject_tags:,titletags.SPLIT
datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
description_limit:1500
[grangerenchanted.com]
## Some sites require login (or login for some rated stories) The
## program can prompt you, or you can save it in config. In
fanficfare/story.py
+17 -3
View File
@@ -465,7 +465,7 @@ class Story(Configurable):
if key == 'dateUpdated' and value:
# Last Update tags for Bill.
self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True)
self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
@@ -757,8 +757,12 @@ class Story(Configurable):
# includelist prevents infinite recursion of include_in_'s
if self.hasConfig("include_in_"+listname) and listname not in includelist:
for k in self.getConfigList("include_in_"+listname):
ldorepl = doreplacements
if k.endswith('.NOREPL'):
k = k[:-len('.NOREPL')]
ldorepl = False
retlist.extend(self.getList(k,removeallentities=False,
doreplacements=doreplacements,includelist=includelist+[listname]))
doreplacements=ldorepl,includelist=includelist+[listname]))
else:
if not self.isList(listname):
@@ -813,7 +817,16 @@ class Story(Configurable):
# metadata all go into dc:subject tags, but only if they are configured.
for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
if name in tags_list:
if name+'.SPLIT' in tags_list:
flist=[]
if isinstance(value,list):
for tag in value:
flist.extend(tag.split(','))
else:
flist.extend(value)
for tag in flist:
subjectset.add(tag)
elif name in tags_list:
if isinstance(value,list):
for tag in value:
subjectset.add(tag)
@@ -911,6 +924,7 @@ class Story(Configurable):
#print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
# apply coverexclusion to explicit covers, too. Primarily for ffnet imageu.
#print("[[[[[\n\n %s %s \n\n]]]]]]]"%(imgurl,coverexclusion))
if cover and coverexclusion and re.search(coverexclusion,imgurl):
return (None,None)