From ea2f64a7fb2d13f0582557342ad471b7b7c1f256 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 25 Jun 2015 13:18:47 -0500 Subject: [PATCH] Add additional features to/for forums(SB&SV) adapters. --- calibre-plugin/plugin-defaults.ini | 103 +++++++++++++++++- .../adapters/adapter_forumsspacebattlescom.py | 62 ++++++++--- fanficfare/adapters/base_adapter.py | 7 ++ fanficfare/defaults.ini | 103 +++++++++++++++++- fanficfare/story.py | 20 +++- 5 files changed, 269 insertions(+), 26 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index da2f078..e8e27a3 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -330,9 +330,10 @@ sort_ships:false ## User-agent user_agent:FFF/2.X -## Virtually all eFiction Base adapters allow downloading the whole story in -## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both -## metadata and chapters can be loaded in one step +## At the time of writing, eFiction Base adapters allow downloading +## the whole story in bulk using the 'Print' feature. If 'bulk_load' +## is set to 'true', both metadata and chapters can be loaded in one +## step bulk_load:true ## Each output format has a section that overrides [defaults] @@ -1040,6 +1041,102 @@ extra_valid_entries:size # don't show twitter icon. cover_exclusion_regexp:/res/css/bir.png +[forums.sufficientvelocity.com] + +cover_exclusion_regexp:/clear.png + +add_to_extratags:ForumFic + +strip_chapter_numbers:false + +# true, false, threadmarksonly +add_chapter_dates:false + +add_to_extra_valid_entries:,titletags +# '.NOREPL' tells the system to *not* apply title's +# in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_titletags:title.NOREPL + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:titletags + +include_metadata_pre: +# only keep titletags with ( or [ in. + titletags=~[\[\(] + +replace_metadata: +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> +# remove anything outside () or [] + titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + titletags=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + titletags=> */ *=>, + titletags=> x =>, +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> + +extra_titlepage_entries: titletags + +## '.SPLIT' teels the system to split by ',' +add_to_include_subject_tags:,titletags.SPLIT + +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +description_limit:1500 + +[forums.spacebattles.com] + +cover_exclusion_regexp:/clear.png + +add_to_extratags:ForumFic + +# true, false, threadmarksonly +add_chapter_dates:false + +strip_chapter_numbers:false + +add_to_extra_valid_entries:,titletags +# '.NOREPL' tells the system to *not* apply title's +# in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_titletags:title.NOREPL + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:titletags + +include_metadata_pre: +# only keep titletags with ( or [ in. + titletags=~[\[\(] + +replace_metadata: +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> +# remove anything outside () or [] + titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + titletags=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + titletags=> */ *=>, + titletags=> x =>, +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> + +extra_titlepage_entries: titletags + +## '.SPLIT' teels the system to split by ',' +add_to_include_subject_tags:,titletags.SPLIT + +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +description_limit:1500 + [grangerenchanted.com] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index 2676757..5351398 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -64,7 +64,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - #self.dateformat = "%Y-%b-%d" + self.dateformat = "%b %d, %Y at %I:%M %p" @staticmethod # must be @staticmethod, don't remove it. def getSiteDomain(): @@ -93,13 +93,13 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): ## Getting the chapter list and the meta data, plus 'is adult' checking. def extractChapterUrlsAndMetadata(self): - url = self.url - logger.info("url: "+url) + useurl = self.url + logger.info("url: "+useurl) try: - (data,opened) = self._fetchUrlOpened(url) - url = opened.geturl() - logger.info("use url: "+url) + (data,opened) = self._fetchUrlOpened(useurl) + useurl = opened.geturl() + logger.info("use useurl: "+useurl) except urllib2.HTTPError, e: if e.code == 404: raise exceptions.StoryDoesNotExist(self.url) @@ -114,13 +114,11 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href']) self.story.addToList('author',a.text) - self.story.addToList('genre','ForumFic') - h1 = soup.find('div',{'class':'titleBar'}).h1 self.story.setMetadata('title',stripHTML(h1)) - - if '#' in url: - anchorid = url.split('#')[1] + + if '#' in useurl: + anchorid = useurl.split('#')[1] soup = soup.find('li',id=anchorid) else: # try threadmarks if no '#' in , require at least 2. @@ -129,17 +127,43 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href'])) markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') if len(markas) > 1: - for (url,name) in [ (x['href'],stripHTML(x)) for x in markas ]: + for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]: + datestr=None + datetag = atag.find_next_sibling('div',{'class':'extra'}).find('span',{'class':'DateTime'}) + if datetag: + datestr = datetag['title'] + else: + datetag = atag.find_next_sibling('div',{'class':'extra'}).find('abbr',{'class':'DateTime'}) + if datetag: + datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring']) + # Apr 24, 2015 at 4:39 AM + # May 1, 2015 at 5:47 AM + datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours. + date = makeDate(datestr, self.dateformat) + if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'): + self.story.setMetadata('datePublished', date) + if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'): + self.story.setMetadata('dateUpdated', date) + + if self.getConfig('add_chapter_dates') in ['true','threadmarksonly']: + name = '%s %s'%(name,date) + self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) # Now go hunting for the 'chapter list'. - firstpost = soup.find('blockquote') # assume first posting contains TOC urls. + bq = soup.find('blockquote') # assume first posting contains TOC urls. + + bq.name='div' + + for iframe in bq.find_all('iframe'): + iframe.extract() # calibre book reader & editor don't like iframes to youtube. + + self.setDescription(useurl,bq) # otherwise, use first post links--include first post since that's if not self.chapterUrls: - #logger.debug("len(firstpost):%s"%len(unicode(firstpost))) - self.chapterUrls.append(("First Post",self.url)) - for (url,name) in [ (x['href'],stripHTML(x)) for x in firstpost.find_all('a') ]: + self.chapterUrls.append(("First Post",useurl)) + for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]: logger.debug("found chapurl:%s"%url) if not url.startswith('http'): url = self.getURLPrefix()+'/'+url @@ -147,8 +171,12 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): if (url.startswith(self.getURLPrefix()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): # brute force way to deal with SB's http->https change when hardcoded http urls. url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix()) - logger.debug("used chapurl:%s"%url) + logger.debug("used chapurl:%s"%(url)) self.chapterUrls.append((name,url)) + if url == useurl and 'First Post' == self.chapterUrls[0][0]: + # remove "First Post" if included in list. + logger.debug("delete dup chapter: %s %s"%self.chapterUrls[0]) + del self.chapterUrls[0] self.story.setMetadata('numChapters',len(self.chapterUrls)) diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 53b6654..a0a0b7f 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -497,6 +497,13 @@ class BaseSiteAdapter(Configurable): def setDescription(self,url,svalue): #print("\n\nsvalue:\n%s\n"%svalue) + strval = u"%s"%svalue # works for either soup or string + if self.hasConfig('description_limit'): + limit = int(self.getConfig('description_limit')) + if limit and len(strval) > limit: + svalue = strval[:limit] + + #print(u"[[[[[\n\n%s\n\n]]]]]]]]"%svalue) # works for either soup or string if self.getConfig('keep_summary_html'): if isinstance(svalue,basestring): # bs4/html5lib add html, header and body tags, which diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 0663c12..52256f8 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -327,9 +327,10 @@ sort_ships:false ## User-agent user_agent:FFF/2.X -## Virtually all eFiction Base adapters allow downloading the whole story in -## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both -## metadata and chapters can be loaded in one step +## At the time of writing, eFiction Base adapters allow downloading +## the whole story in bulk using the 'Print' feature. If 'bulk_load' +## is set to 'true', both metadata and chapters can be loaded in one +## step bulk_load:true ## Each output format has a section that overrides [defaults] @@ -1026,6 +1027,102 @@ extra_valid_entries:size # don't show twitter icon. cover_exclusion_regexp:/res/css/bir.png +[forums.sufficientvelocity.com] + +cover_exclusion_regexp:/clear.png + +add_to_extratags:ForumFic + +strip_chapter_numbers:false + +# true, false, threadmarksonly +add_chapter_dates:false + +add_to_extra_valid_entries:,titletags +# '.NOREPL' tells the system to *not* apply title's +# in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_titletags:title.NOREPL + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:titletags + +include_metadata_pre: +# only keep titletags with ( or [ in. + titletags=~[\[\(] + +replace_metadata: +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> +# remove anything outside () or [] + titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + titletags=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + titletags=> */ *=>, + titletags=> x =>, +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> + +extra_titlepage_entries: titletags + +## '.SPLIT' teels the system to split by ',' +add_to_include_subject_tags:,titletags.SPLIT + +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +description_limit:1500 + +[forums.spacebattles.com] + +cover_exclusion_regexp:/clear.png + +add_to_extratags:ForumFic + +# true, false, threadmarksonly +add_chapter_dates:false + +strip_chapter_numbers:false + +add_to_extra_valid_entries:,titletags +# '.NOREPL' tells the system to *not* apply title's +# in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_titletags:title.NOREPL + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:titletags + +include_metadata_pre: +# only keep titletags with ( or [ in. + titletags=~[\[\(] + +replace_metadata: +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> +# remove anything outside () or [] + titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + titletags=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + titletags=> */ *=>, + titletags=> x =>, +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> + +extra_titlepage_entries: titletags + +## '.SPLIT' teels the system to split by ',' +add_to_include_subject_tags:,titletags.SPLIT + +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +description_limit:1500 + [grangerenchanted.com] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In diff --git a/fanficfare/story.py b/fanficfare/story.py index 19c30ef..2e7c2ae 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -465,7 +465,7 @@ class Story(Configurable): if key == 'dateUpdated' and value: # Last Update tags for Bill. - self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m")) + self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True) self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d")) @@ -757,8 +757,12 @@ class Story(Configurable): # includelist prevents infinite recursion of include_in_'s if self.hasConfig("include_in_"+listname) and listname not in includelist: for k in self.getConfigList("include_in_"+listname): + ldorepl = doreplacements + if k.endswith('.NOREPL'): + k = k[:-len('.NOREPL')] + ldorepl = False retlist.extend(self.getList(k,removeallentities=False, - doreplacements=doreplacements,includelist=includelist+[listname])) + doreplacements=ldorepl,includelist=includelist+[listname])) else: if not self.isList(listname): @@ -813,7 +817,16 @@ class Story(Configurable): # metadata all go into dc:subject tags, but only if they are configured. for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems(): - if name in tags_list: + if name+'.SPLIT' in tags_list: + flist=[] + if isinstance(value,list): + for tag in value: + flist.extend(tag.split(',')) + else: + flist.extend(value) + for tag in flist: + subjectset.add(tag) + elif name in tags_list: if isinstance(value,list): for tag in value: subjectset.add(tag) @@ -911,6 +924,7 @@ class Story(Configurable): #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl)) # apply coverexclusion to explicit covers, too. Primarily for ffnet imageu. + #print("[[[[[\n\n %s %s \n\n]]]]]]]"%(imgurl,coverexclusion)) if cover and coverexclusion and re.search(coverexclusion,imgurl): return (None,None)