From f62172b40ac18f392ed0b675f42c49a3cae7ab7f Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 25 May 2015 14:19:59 -0500 Subject: [PATCH 01/32] Add semi-support for forums.spacebattles.com. --- fanficfare/adapters/__init__.py | 1 + .../adapters/adapter_forumsspacebattlescom.py | 159 ++++++++++++++++++ fanficfare/adapters/base_adapter.py | 52 +++++- 3 files changed, 203 insertions(+), 9 deletions(-) create mode 100644 fanficfare/adapters/adapter_forumsspacebattlescom.py diff --git a/fanficfare/adapters/__init__.py b/fanficfare/adapters/__init__.py index 606e0c5..d6ff27a 100644 --- a/fanficfare/adapters/__init__.py +++ b/fanficfare/adapters/__init__.py @@ -135,6 +135,7 @@ import adapter_fanfictionjunkiesde import adapter_devianthearts import adapter_tgstorytimecom import adapter_itcouldhappennet +import adapter_forumsspacebattlescom ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py new file mode 100644 index 0000000..63d2668 --- /dev/null +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 FanFicFare team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return ForumsSpacebattlesComAdapter + +logger = logging.getLogger(__name__) + +class ForumsSpacebattlesComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["utf8", + "Windows-1252"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2]) + + + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/threads/'+self.story.getMetadata('storyId')+'/') + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','fsb') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + #self.dateformat = "%Y-%b-%d" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'forums.spacebattles.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/threads/some-story-name.123456/" + + def getSiteURLPattern(self): + # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770 + # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs. + return r"http://"+re.escape(self.getSiteDomain())+r"/threads/(.+\.)?(?P\d+)/" + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + logger.info("url: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = self.make_soup(data) + + a = soup.find('h3',{'class':'userText'}).find('a') + self.story.addToList('authorId',a['href'].split('/')[1]) + self.story.addToList('authorUrl','http://'+self.getSiteDomain()+'/'+a['href']) + self.story.addToList('author',a.text) + + self.story.addToList('genre','ForumFic') + + h1 = soup.find('div',{'class':'titleBar'}).h1 + self.story.setMetadata('title',stripHTML(h1)) + + # Now go hunting for the 'chapter list'. + firstpost = soup.find('blockquote') # assume first posting contains TOC urls. + + # try threadmarks first, require at least 2. + threadmarksa = soup.find('a',{'class':'threadmarksTrigger'}) + if threadmarksa: + soupmarks = self.make_soup(self._fetchUrl('http://'+self.getSiteDomain()+'/'+threadmarksa['href'])) + markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') + if len(markas) > 1: + for (url,name) in [ (x['href'],stripHTML(x)) for x in markas ]: + self.chapterUrls.append((name,'http://'+self.getSiteDomain()+'/'+url)) + + # otherwise, use first post links--include first post since that's + if not self.chapterUrls: + logger.debug("len(firstpost):%s"%len(unicode(firstpost))) + self.chapterUrls.append(("First Post",self.url)) + for (url,name) in [ (x['href'],stripHTML(x)) for x in firstpost.find_all('a') ]: + if not url.startswith('http'): + url = 'http://'+self.getSiteDomain()+'/'+url + + if url.startswith('http://'+self.getSiteDomain()) and ('/posts/' in url or '/threads/' in url): + self.chapterUrls.append((name,url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # grab the text for an individual chapter. + def getChapterText(self, url): + logger.debug('Getting chapter text from: %s' % url) + + (data,opened) = self._fetchUrlOpened(url) + url = opened.geturl() + logger.debug("chapter URL redirected to: %s"%url) + + soup = self.make_soup(data) + + if '#' in url: + anchorid = url.split('#')[1] + soup = soup.find('li',id=anchorid) + bq = soup.find('blockquote') + + bq.name='div' + + return self.utf8FromSoup(url,bq) diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 2b76d69..61ca68a 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -279,6 +279,16 @@ class BaseSiteAdapter(Configurable): parameters=None, extrasleep=None, usecache=True): + + return self._fetchUrlRawOpened(url, + parameters, + extrasleep, + usecache)[0] + + def _fetchUrlRawOpened(self, url, + parameters=None, + extrasleep=None, + usecache=True): ''' When should cache be cleared or not used? logins... @@ -289,16 +299,24 @@ class BaseSiteAdapter(Configurable): cachekey=self._get_cachekey(url, parameters) if usecache and self._has_cachekey(cachekey): logger.debug("#####################################\npagecache HIT: %s"%cachekey) - return self._get_from_pagecache(cachekey) + data = self._get_from_pagecache(cachekey) + class FakeOpened: + def __init__(self,data,url): + self.data=data + self.url=url + def geturl(self): return self.url + def read(self): return self.data + return (data,FakeOpened(data,cachekey)) logger.debug("#####################################\npagecache MISS: %s"%cachekey) self.do_sleep(extrasleep) if parameters != None: - data = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read() + opened = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))) else: - data = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read() + opened = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))) + data = opened.read() self._set_to_pagecache(cachekey,data) - return data + return (data,opened) def set_sleep(self,val): logger.debug("\n===========\n set sleep time %s\n==========="%val) @@ -312,20 +330,30 @@ class BaseSiteAdapter(Configurable): elif self.getConfig('slow_down_sleep_time'): time.sleep(float(self.getConfig('slow_down_sleep_time'))) - # parameters is a dict() def _fetchUrl(self, url, parameters=None, usecache=True, extrasleep=None): + return self._fetchUrlOpened(url, + parameters, + usecache, + extrasleep)[0] + + # parameters is a dict() + def _fetchUrlOpened(self, url, + parameters=None, + usecache=True, + extrasleep=None): excpt=None for sleeptime in [0, 0.5, 4, 9]: time.sleep(sleeptime) try: - return self._decode(self._fetchUrlRaw(url, + (data,opened)=self._fetchUrlRawOpened(url, parameters=parameters, usecache=usecache, - extrasleep=extrasleep)) + extrasleep=extrasleep) + return (self._decode(data),opened) except u2.HTTPError, he: excpt=he if he.code == 404: @@ -399,7 +427,10 @@ class BaseSiteAdapter(Configurable): self.doExtractChapterUrlsAndMetadata(get_cover=get_cover) if not self.story.getMetadataRaw('dateUpdated'): - self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished')) + if self.story.getMetadataRaw('datePublished'): + self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished')) + else: + self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated')) self.metadataDone = True return self.story @@ -409,7 +440,10 @@ class BaseSiteAdapter(Configurable): self.story.load_html_metadata(metahtml) self.metadataDone = True if not self.story.getMetadataRaw('dateUpdated'): - self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished')) + if self.story.getMetadataRaw('datePublished'): + self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished')) + else: + self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated')) def hookForUpdates(self,chaptercount): "Usually not needed." From a11d4729bd0b5409134e9b97de59a1984de7b6ec Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 28 May 2015 19:34:19 -0500 Subject: [PATCH 02/32] Add update chapters by remembered URLs feature. --- calibre-plugin/jobs.py | 3 ++- fanficfare/adapters/base_adapter.py | 11 +++++++++-- fanficfare/cli.py | 3 ++- fanficfare/epubutils.py | 8 +++++++- fanficfare/writers/writer_epub.py | 4 +++- 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index 5b7788c..91993a6 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -190,7 +190,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, - adapter.logfile) = get_update_data(book['epub_for_update'])[0:7] + adapter.logfile, + adapter.oldchaptersmap) = get_update_data(book['epub_for_update'])[0:8] # dup handling from fff_plugin needed for anthology updates. if options['collision'] == UPDATE: diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 61ca68a..c1f754a 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -104,6 +104,7 @@ class BaseSiteAdapter(Configurable): self.chapterFirst = None self.chapterLast = None self.oldchapters = None + self.oldchaptersmap = None self.oldimgs = None self.oldcover = None # (data of existing cover html, data of existing cover image) self.calibrebookmark = None @@ -386,11 +387,17 @@ class BaseSiteAdapter(Configurable): removeEntities(title), None) else: - if self.oldchapters and index < len(self.oldchapters): + data = None + if self.oldchaptersmap: + if url in self.oldchaptersmap: + data = self.utf8FromSoup(None, + self.oldchaptersmap[url], + partial(cachedfetch,self._fetchUrlRaw,self.oldimgs)) + elif self.oldchapters and index < len(self.oldchapters): data = self.utf8FromSoup(None, self.oldchapters[index], partial(cachedfetch,self._fetchUrlRaw,self.oldimgs)) - else: + if not data: data = self.getChapterText(url) self.story.addChapter(url, removeEntities(title), diff --git a/fanficfare/cli.py b/fanficfare/cli.py index a14f9de..0856ce7 100644 --- a/fanficfare/cli.py +++ b/fanficfare/cli.py @@ -324,7 +324,8 @@ def do_download(arg, adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, - adapter.logfile) = (get_update_data(output_filename))[0:7] + adapter.logfile, + adapter.oldchaptersmap) = (get_update_data(output_filename))[0:8] print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount) diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index 0436656..5977876 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -90,6 +90,7 @@ def get_update_data(inputio, filecount = 0 soups = [] # list of xhmtl blocks + urlsoups = {} # map of xhtml blocks by url images = {} # dict() longdesc->data if getfilecount: # spin through the manifest--only place there are item tags. @@ -136,6 +137,11 @@ def get_update_data(inputio, for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}): skip.extract() + + chapa = soup.find('a',{'class':'chapterurl'}) + if chapa: + urlsoups[chapa['href']] = soup + chapa.extract() soups.append(soup) @@ -148,7 +154,7 @@ def get_update_data(inputio, #for k in images.keys(): #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k]))) - return (source,filecount,soups,images,oldcover,calibrebookmark,logfile) + return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups) def get_path_part(n): relpath = os.path.dirname(n) diff --git a/fanficfare/writers/writer_epub.py b/fanficfare/writers/writer_epub.py index abbb646..52deae9 100644 --- a/fanficfare/writers/writer_epub.py +++ b/fanficfare/writers/writer_epub.py @@ -654,7 +654,9 @@ div { margin: 0pt; padding: 0pt; } if html: logger.debug('Writing chapter text for: %s' % title) vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} - fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals) + fullhtml = CHAPTER_START.substitute(vals) + \ + '' + \ + html + CHAPTER_END.substitute(vals) # ffnet(& maybe others) gives the whole chapter text # as one line. This causes problems for nook(at # least) when the chapter size starts getting big From 69ae58686e87fd60e590e4795dece630a617a9e3 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Sat, 30 May 2015 16:18:36 -0500 Subject: [PATCH 03/32] Normalize story URLs from email before comparing to Reject List. --- calibre-plugin/fff_plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/calibre-plugin/fff_plugin.py b/calibre-plugin/fff_plugin.py index f9da383..1f15ccf 100644 --- a/calibre-plugin/fff_plugin.py +++ b/calibre-plugin/fff_plugin.py @@ -447,7 +447,8 @@ class FanFicFarePlugin(InterfaceAction): prefs['imapmarkread'],) reject_list=set() if prefs['auto_reject_from_email']: - reject_list = set([x for x in url_list if rejecturllist.check(x)]) + # need to normalize for reject list. + reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURLSite(x)[0])]) url_list = url_list - reject_list self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000) From fcc5c1424d9884b7a5d336040663076c16fa51bf Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 1 Jun 2015 09:31:00 -0500 Subject: [PATCH 04/32] Change forums.spacebattles.com to https. --- .../adapters/adapter_forumsspacebattlescom.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index 63d2668..b735f5a 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -53,7 +53,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): self.story.setMetadata('storyId',m.group('id')) # normalized story URL. - self._setURL('http://' + self.getSiteDomain() + '/threads/'+self.story.getMetadata('storyId')+'/') + self._setURL('https://' + self.getSiteDomain() + '/threads/'+self.story.getMetadata('storyId')+'/') else: raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), @@ -73,12 +73,12 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): @classmethod def getSiteExampleURLs(cls): - return "http://"+cls.getSiteDomain()+"/threads/some-story-name.123456/" + return "https://"+cls.getSiteDomain()+"/threads/some-story-name.123456/" def getSiteURLPattern(self): # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770 # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs. - return r"http://"+re.escape(self.getSiteDomain())+r"/threads/(.+\.)?(?P\d+)/" + return r"https?://"+re.escape(self.getSiteDomain())+r"/threads/(.+\.)?(?P\d+)/" def use_pagecache(self): ''' @@ -106,7 +106,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): a = soup.find('h3',{'class':'userText'}).find('a') self.story.addToList('authorId',a['href'].split('/')[1]) - self.story.addToList('authorUrl','http://'+self.getSiteDomain()+'/'+a['href']) + self.story.addToList('authorUrl','https://'+self.getSiteDomain()+'/'+a['href']) self.story.addToList('author',a.text) self.story.addToList('genre','ForumFic') @@ -120,21 +120,21 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): # try threadmarks first, require at least 2. threadmarksa = soup.find('a',{'class':'threadmarksTrigger'}) if threadmarksa: - soupmarks = self.make_soup(self._fetchUrl('http://'+self.getSiteDomain()+'/'+threadmarksa['href'])) + soupmarks = self.make_soup(self._fetchUrl('https://'+self.getSiteDomain()+'/'+threadmarksa['href'])) markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') if len(markas) > 1: for (url,name) in [ (x['href'],stripHTML(x)) for x in markas ]: - self.chapterUrls.append((name,'http://'+self.getSiteDomain()+'/'+url)) + self.chapterUrls.append((name,'https://'+self.getSiteDomain()+'/'+url)) # otherwise, use first post links--include first post since that's if not self.chapterUrls: - logger.debug("len(firstpost):%s"%len(unicode(firstpost))) + #logger.debug("len(firstpost):%s"%len(unicode(firstpost))) self.chapterUrls.append(("First Post",self.url)) for (url,name) in [ (x['href'],stripHTML(x)) for x in firstpost.find_all('a') ]: if not url.startswith('http'): - url = 'http://'+self.getSiteDomain()+'/'+url + url = 'https://'+self.getSiteDomain()+'/'+url - if url.startswith('http://'+self.getSiteDomain()) and ('/posts/' in url or '/threads/' in url): + if (url.startswith('https://'+self.getSiteDomain()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): self.chapterUrls.append((name,url)) self.story.setMetadata('numChapters',len(self.chapterUrls)) From a9aa7ba5052ae92e137deb384db6221259ce46c5 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 4 Jun 2015 08:53:22 -0500 Subject: [PATCH 05/32] Add forums.sufficientvelocity.com as a child of SB adapter. --- fanficfare/adapters/__init__.py | 1 + .../adapters/adapter_forumsspacebattlescom.py | 23 ++++++----- .../adapter_forumssufficientvelocitycom.py | 39 +++++++++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 fanficfare/adapters/adapter_forumssufficientvelocitycom.py diff --git a/fanficfare/adapters/__init__.py b/fanficfare/adapters/__init__.py index d6ff27a..301cd9c 100644 --- a/fanficfare/adapters/__init__.py +++ b/fanficfare/adapters/__init__.py @@ -136,6 +136,7 @@ import adapter_devianthearts import adapter_tgstorytimecom import adapter_itcouldhappennet import adapter_forumsspacebattlescom +import adapter_forumssufficientvelocitycom ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index b735f5a..2e5e5bf 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -53,7 +53,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): self.story.setMetadata('storyId',m.group('id')) # normalized story URL. - self._setURL('https://' + self.getSiteDomain() + '/threads/'+self.story.getMetadata('storyId')+'/') + self._setURL(self.getURLPrefix() + '/threads/'+self.story.getMetadata('storyId')+'/') else: raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), @@ -71,13 +71,16 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): # The site domain. Does have www here, if it uses it. return 'forums.spacebattles.com' + @classmethod + def getURLPrefix(cls): + # The site domain. Does have www here, if it uses it. + return 'https://' + cls.getSiteDomain() + @classmethod def getSiteExampleURLs(cls): - return "https://"+cls.getSiteDomain()+"/threads/some-story-name.123456/" + return cls.getURLPrefix()+"/threads/some-story-name.123456/" def getSiteURLPattern(self): - # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770 - # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs. return r"https?://"+re.escape(self.getSiteDomain())+r"/threads/(.+\.)?(?P\d+)/" def use_pagecache(self): @@ -106,7 +109,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): a = soup.find('h3',{'class':'userText'}).find('a') self.story.addToList('authorId',a['href'].split('/')[1]) - self.story.addToList('authorUrl','https://'+self.getSiteDomain()+'/'+a['href']) + self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href']) self.story.addToList('author',a.text) self.story.addToList('genre','ForumFic') @@ -120,11 +123,11 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): # try threadmarks first, require at least 2. threadmarksa = soup.find('a',{'class':'threadmarksTrigger'}) if threadmarksa: - soupmarks = self.make_soup(self._fetchUrl('https://'+self.getSiteDomain()+'/'+threadmarksa['href'])) + soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href'])) markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') if len(markas) > 1: for (url,name) in [ (x['href'],stripHTML(x)) for x in markas ]: - self.chapterUrls.append((name,'https://'+self.getSiteDomain()+'/'+url)) + self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) # otherwise, use first post links--include first post since that's if not self.chapterUrls: @@ -132,9 +135,11 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): self.chapterUrls.append(("First Post",self.url)) for (url,name) in [ (x['href'],stripHTML(x)) for x in firstpost.find_all('a') ]: if not url.startswith('http'): - url = 'https://'+self.getSiteDomain()+'/'+url + url = self.getURLPrefix()+'/'+url - if (url.startswith('https://'+self.getSiteDomain()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): + if (url.startswith(self.getURLPrefix()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): + # brute force way to deal with SB's http->https change when hardcoded http urls. + url.replace('http://'+self.getSiteDomain(),self.getURLPrefix()) self.chapterUrls.append((name,url)) self.story.setMetadata('numChapters',len(self.chapterUrls)) diff --git a/fanficfare/adapters/adapter_forumssufficientvelocitycom.py b/fanficfare/adapters/adapter_forumssufficientvelocitycom.py new file mode 100644 index 0000000..f16785e --- /dev/null +++ b/fanficfare/adapters/adapter_forumssufficientvelocitycom.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 FanFicFare team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from adapter_forumsspacebattlescom import ForumsSpacebattlesComAdapter + +def getClass(): + return ForumsSufficientVelocityComAdapter + +class ForumsSufficientVelocityComAdapter(ForumsSpacebattlesComAdapter): + + def __init__(self, config, url): + ForumsSpacebattlesComAdapter.__init__(self, config, url) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','fsv') + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'forums.sufficientvelocity.com' + + @classmethod + def getURLPrefix(cls): + # The site domain. Does have www here, if it uses it. + return 'http://' + cls.getSiteDomain() From 3428c18c471556624cbdffbce31b0d7d85b3a973 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 4 Jun 2015 09:26:38 -0500 Subject: [PATCH 06/32] Fix for hardcoded http:// urls being redirected to https:// urls on SB --- fanficfare/adapters/adapter_forumsspacebattlescom.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index 2e5e5bf..a5621fb 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -134,12 +134,14 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): #logger.debug("len(firstpost):%s"%len(unicode(firstpost))) self.chapterUrls.append(("First Post",self.url)) for (url,name) in [ (x['href'],stripHTML(x)) for x in firstpost.find_all('a') ]: + logger.debug("found chapurl:%s"%url) if not url.startswith('http'): url = self.getURLPrefix()+'/'+url if (url.startswith(self.getURLPrefix()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): # brute force way to deal with SB's http->https change when hardcoded http urls. - url.replace('http://'+self.getSiteDomain(),self.getURLPrefix()) + url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix()) + logger.debug("used chapurl:%s"%url) self.chapterUrls.append((name,url)) self.story.setMetadata('numChapters',len(self.chapterUrls)) From 0246ecafcf83a3568560afbcb275d9087f0095dc Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 5 Jun 2015 10:55:19 -0500 Subject: [PATCH 07/32] Handle dup chapter URLs with redirect better in cache. --- fanficfare/adapters/base_adapter.py | 38 +++++++---------------------- fanficfare/epubutils.py | 2 +- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index c1f754a..53b6654 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -141,28 +141,6 @@ class BaseSiteAdapter(Configurable): ''' self.get_cookiejar().load(filename, ignore_discard=True, ignore_expires=True) - # def save_cookiejar(self,filename): - # ''' - # Assumed to be a FileCookieJar if self.cookiejar set. - # Takes file *name*. - # ''' - # self.get_cookiejar().save(filename, ignore_discard=True, ignore_expires=True) - - # def save_pagecache(self,filename): - # ''' - # Writes pickle of pagecache to file *name* - # ''' - # with open(filename, 'wb') as f: - # pickle.dump(self.get_pagecache(), - # f,protocol=pickle.HIGHEST_PROTOCOL) - - # def load_pagecache(self,filename): - # ''' - # Reads pickle of pagecache from file *name* - # ''' - # with open(filename, 'rb') as f: - # self.set_pagecache(pickle.load(f)) - def get_pagecache(self): return self.pagecache @@ -186,9 +164,9 @@ class BaseSiteAdapter(Configurable): else: return None - def _set_to_pagecache(self,cachekey,data): + def _set_to_pagecache(self,cachekey,data,redirectedurl): if self.use_pagecache(): - self.get_pagecache()[cachekey] = data + self.get_pagecache()[cachekey] = (data,redirectedurl) def use_pagecache(self): ''' @@ -258,7 +236,8 @@ class BaseSiteAdapter(Configurable): cachekey=self._get_cachekey(url, parameters, headers) if usecache and self._has_cachekey(cachekey): logger.debug("#####################################\npagecache HIT: %s"%cachekey) - return self._get_from_pagecache(cachekey) + data,redirecturl = self._get_from_pagecache(cachekey) + return data logger.debug("#####################################\npagecache MISS: %s"%cachekey) self.do_sleep(extrasleep) @@ -273,7 +252,7 @@ class BaseSiteAdapter(Configurable): data=urllib.urlencode(parameters), headers=headers) data = self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()) - self._set_to_pagecache(cachekey,data) + self._set_to_pagecache(cachekey,data,url) return data def _fetchUrlRaw(self, url, @@ -300,14 +279,14 @@ class BaseSiteAdapter(Configurable): cachekey=self._get_cachekey(url, parameters) if usecache and self._has_cachekey(cachekey): logger.debug("#####################################\npagecache HIT: %s"%cachekey) - data = self._get_from_pagecache(cachekey) + data,redirecturl = self._get_from_pagecache(cachekey) class FakeOpened: def __init__(self,data,url): self.data=data self.url=url def geturl(self): return self.url def read(self): return self.data - return (data,FakeOpened(data,cachekey)) + return (data,FakeOpened(data,redirecturl)) logger.debug("#####################################\npagecache MISS: %s"%cachekey) self.do_sleep(extrasleep) @@ -316,7 +295,8 @@ class BaseSiteAdapter(Configurable): else: opened = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))) data = opened.read() - self._set_to_pagecache(cachekey,data) + self._set_to_pagecache(cachekey,data,opened.url) + return (data,opened) def set_sleep(self,val): diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index 5977876..6e51164 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -139,7 +139,7 @@ def get_update_data(inputio, skip.extract() chapa = soup.find('a',{'class':'chapterurl'}) - if chapa: + if chapa and chapa['href'] not in urlsoups: # keep first found if more than one. urlsoups[chapa['href']] = soup chapa.extract() From d2535ef12ba9afdc3c70f17a70d57673f0d02829 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 8 Jun 2015 11:13:17 -0500 Subject: [PATCH 08/32] Exclude iframe tags in forums stories. --- fanficfare/adapters/adapter_forumsspacebattlescom.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index a5621fb..eadfd8c 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -163,4 +163,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): bq.name='div' + for iframe in bq.find_all('iframe'): + iframe.extract() # calibre book reader & editor don't like iframes to youtube. + return self.utf8FromSoup(url,bq) From c3c4fb83449b024336d633232c033596f8e38677 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 10 Jun 2015 23:44:36 -0500 Subject: [PATCH 09/32] Allow forums URLs to point to non-first posts for index. --- .../adapters/adapter_forumsspacebattlescom.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index eadfd8c..2676757 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -51,9 +51,9 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): m = re.match(self.getSiteURLPattern(),url) if m: self.story.setMetadata('storyId',m.group('id')) - + # normalized story URL. - self._setURL(self.getURLPrefix() + '/threads/'+self.story.getMetadata('storyId')+'/') + self._setURL(self.getURLPrefix() + '/'+m.group('tp')+'/'+self.story.getMetadata('storyId')+'/') else: raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), @@ -81,7 +81,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): return cls.getURLPrefix()+"/threads/some-story-name.123456/" def getSiteURLPattern(self): - return r"https?://"+re.escape(self.getSiteDomain())+r"/threads/(.+\.)?(?P\d+)/" + return r"https?://"+re.escape(self.getSiteDomain())+r"/(?Pthreads|posts)/(.+\.)?(?P\d+)/" def use_pagecache(self): ''' @@ -97,7 +97,9 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): logger.info("url: "+url) try: - data = self._fetchUrl(url) + (data,opened) = self._fetchUrlOpened(url) + url = opened.geturl() + logger.info("use url: "+url) except urllib2.HTTPError, e: if e.code == 404: raise exceptions.StoryDoesNotExist(self.url) @@ -117,18 +119,22 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): h1 = soup.find('div',{'class':'titleBar'}).h1 self.story.setMetadata('title',stripHTML(h1)) + if '#' in url: + anchorid = url.split('#')[1] + soup = soup.find('li',id=anchorid) + else: + # try threadmarks if no '#' in , require at least 2. + threadmarksa = soup.find('a',{'class':'threadmarksTrigger'}) + if threadmarksa: + soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href'])) + markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') + if len(markas) > 1: + for (url,name) in [ (x['href'],stripHTML(x)) for x in markas ]: + self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) + # Now go hunting for the 'chapter list'. firstpost = soup.find('blockquote') # assume first posting contains TOC urls. - # try threadmarks first, require at least 2. - threadmarksa = soup.find('a',{'class':'threadmarksTrigger'}) - if threadmarksa: - soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href'])) - markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') - if len(markas) > 1: - for (url,name) in [ (x['href'],stripHTML(x)) for x in markas ]: - self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) - # otherwise, use first post links--include first post since that's if not self.chapterUrls: #logger.debug("len(firstpost):%s"%len(unicode(firstpost))) From ea2f64a7fb2d13f0582557342ad471b7b7c1f256 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 25 Jun 2015 13:18:47 -0500 Subject: [PATCH 10/32] Add additional features to/for forums(SB&SV) adapters. --- calibre-plugin/plugin-defaults.ini | 103 +++++++++++++++++- .../adapters/adapter_forumsspacebattlescom.py | 62 ++++++++--- fanficfare/adapters/base_adapter.py | 7 ++ fanficfare/defaults.ini | 103 +++++++++++++++++- fanficfare/story.py | 20 +++- 5 files changed, 269 insertions(+), 26 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index da2f078..e8e27a3 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -330,9 +330,10 @@ sort_ships:false ## User-agent user_agent:FFF/2.X -## Virtually all eFiction Base adapters allow downloading the whole story in -## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both -## metadata and chapters can be loaded in one step +## At the time of writing, eFiction Base adapters allow downloading +## the whole story in bulk using the 'Print' feature. If 'bulk_load' +## is set to 'true', both metadata and chapters can be loaded in one +## step bulk_load:true ## Each output format has a section that overrides [defaults] @@ -1040,6 +1041,102 @@ extra_valid_entries:size # don't show twitter icon. cover_exclusion_regexp:/res/css/bir.png +[forums.sufficientvelocity.com] + +cover_exclusion_regexp:/clear.png + +add_to_extratags:ForumFic + +strip_chapter_numbers:false + +# true, false, threadmarksonly +add_chapter_dates:false + +add_to_extra_valid_entries:,titletags +# '.NOREPL' tells the system to *not* apply title's +# in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_titletags:title.NOREPL + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:titletags + +include_metadata_pre: +# only keep titletags with ( or [ in. + titletags=~[\[\(] + +replace_metadata: +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> +# remove anything outside () or [] + titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + titletags=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + titletags=> */ *=>, + titletags=> x =>, +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> + +extra_titlepage_entries: titletags + +## '.SPLIT' teels the system to split by ',' +add_to_include_subject_tags:,titletags.SPLIT + +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +description_limit:1500 + +[forums.spacebattles.com] + +cover_exclusion_regexp:/clear.png + +add_to_extratags:ForumFic + +# true, false, threadmarksonly +add_chapter_dates:false + +strip_chapter_numbers:false + +add_to_extra_valid_entries:,titletags +# '.NOREPL' tells the system to *not* apply title's +# in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_titletags:title.NOREPL + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:titletags + +include_metadata_pre: +# only keep titletags with ( or [ in. + titletags=~[\[\(] + +replace_metadata: +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> +# remove anything outside () or [] + titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + titletags=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + titletags=> */ *=>, + titletags=> x =>, +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> + +extra_titlepage_entries: titletags + +## '.SPLIT' teels the system to split by ',' +add_to_include_subject_tags:,titletags.SPLIT + +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +description_limit:1500 + [grangerenchanted.com] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index 2676757..5351398 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -64,7 +64,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - #self.dateformat = "%Y-%b-%d" + self.dateformat = "%b %d, %Y at %I:%M %p" @staticmethod # must be @staticmethod, don't remove it. def getSiteDomain(): @@ -93,13 +93,13 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): ## Getting the chapter list and the meta data, plus 'is adult' checking. def extractChapterUrlsAndMetadata(self): - url = self.url - logger.info("url: "+url) + useurl = self.url + logger.info("url: "+useurl) try: - (data,opened) = self._fetchUrlOpened(url) - url = opened.geturl() - logger.info("use url: "+url) + (data,opened) = self._fetchUrlOpened(useurl) + useurl = opened.geturl() + logger.info("use useurl: "+useurl) except urllib2.HTTPError, e: if e.code == 404: raise exceptions.StoryDoesNotExist(self.url) @@ -114,13 +114,11 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href']) self.story.addToList('author',a.text) - self.story.addToList('genre','ForumFic') - h1 = soup.find('div',{'class':'titleBar'}).h1 self.story.setMetadata('title',stripHTML(h1)) - - if '#' in url: - anchorid = url.split('#')[1] + + if '#' in useurl: + anchorid = useurl.split('#')[1] soup = soup.find('li',id=anchorid) else: # try threadmarks if no '#' in , require at least 2. @@ -129,17 +127,43 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href'])) markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') if len(markas) > 1: - for (url,name) in [ (x['href'],stripHTML(x)) for x in markas ]: + for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]: + datestr=None + datetag = atag.find_next_sibling('div',{'class':'extra'}).find('span',{'class':'DateTime'}) + if datetag: + datestr = datetag['title'] + else: + datetag = atag.find_next_sibling('div',{'class':'extra'}).find('abbr',{'class':'DateTime'}) + if datetag: + datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring']) + # Apr 24, 2015 at 4:39 AM + # May 1, 2015 at 5:47 AM + datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours. + date = makeDate(datestr, self.dateformat) + if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'): + self.story.setMetadata('datePublished', date) + if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'): + self.story.setMetadata('dateUpdated', date) + + if self.getConfig('add_chapter_dates') in ['true','threadmarksonly']: + name = '%s %s'%(name,date) + self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) # Now go hunting for the 'chapter list'. - firstpost = soup.find('blockquote') # assume first posting contains TOC urls. + bq = soup.find('blockquote') # assume first posting contains TOC urls. + + bq.name='div' + + for iframe in bq.find_all('iframe'): + iframe.extract() # calibre book reader & editor don't like iframes to youtube. + + self.setDescription(useurl,bq) # otherwise, use first post links--include first post since that's if not self.chapterUrls: - #logger.debug("len(firstpost):%s"%len(unicode(firstpost))) - self.chapterUrls.append(("First Post",self.url)) - for (url,name) in [ (x['href'],stripHTML(x)) for x in firstpost.find_all('a') ]: + self.chapterUrls.append(("First Post",useurl)) + for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]: logger.debug("found chapurl:%s"%url) if not url.startswith('http'): url = self.getURLPrefix()+'/'+url @@ -147,8 +171,12 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): if (url.startswith(self.getURLPrefix()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): # brute force way to deal with SB's http->https change when hardcoded http urls. url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix()) - logger.debug("used chapurl:%s"%url) + logger.debug("used chapurl:%s"%(url)) self.chapterUrls.append((name,url)) + if url == useurl and 'First Post' == self.chapterUrls[0][0]: + # remove "First Post" if included in list. + logger.debug("delete dup chapter: %s %s"%self.chapterUrls[0]) + del self.chapterUrls[0] self.story.setMetadata('numChapters',len(self.chapterUrls)) diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 53b6654..a0a0b7f 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -497,6 +497,13 @@ class BaseSiteAdapter(Configurable): def setDescription(self,url,svalue): #print("\n\nsvalue:\n%s\n"%svalue) + strval = u"%s"%svalue # works for either soup or string + if self.hasConfig('description_limit'): + limit = int(self.getConfig('description_limit')) + if limit and len(strval) > limit: + svalue = strval[:limit] + + #print(u"[[[[[\n\n%s\n\n]]]]]]]]"%svalue) # works for either soup or string if self.getConfig('keep_summary_html'): if isinstance(svalue,basestring): # bs4/html5lib add html, header and body tags, which diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 0663c12..52256f8 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -327,9 +327,10 @@ sort_ships:false ## User-agent user_agent:FFF/2.X -## Virtually all eFiction Base adapters allow downloading the whole story in -## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both -## metadata and chapters can be loaded in one step +## At the time of writing, eFiction Base adapters allow downloading +## the whole story in bulk using the 'Print' feature. If 'bulk_load' +## is set to 'true', both metadata and chapters can be loaded in one +## step bulk_load:true ## Each output format has a section that overrides [defaults] @@ -1026,6 +1027,102 @@ extra_valid_entries:size # don't show twitter icon. cover_exclusion_regexp:/res/css/bir.png +[forums.sufficientvelocity.com] + +cover_exclusion_regexp:/clear.png + +add_to_extratags:ForumFic + +strip_chapter_numbers:false + +# true, false, threadmarksonly +add_chapter_dates:false + +add_to_extra_valid_entries:,titletags +# '.NOREPL' tells the system to *not* apply title's +# in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_titletags:title.NOREPL + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:titletags + +include_metadata_pre: +# only keep titletags with ( or [ in. + titletags=~[\[\(] + +replace_metadata: +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> +# remove anything outside () or [] + titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + titletags=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + titletags=> */ *=>, + titletags=> x =>, +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> + +extra_titlepage_entries: titletags + +## '.SPLIT' teels the system to split by ',' +add_to_include_subject_tags:,titletags.SPLIT + +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +description_limit:1500 + +[forums.spacebattles.com] + +cover_exclusion_regexp:/clear.png + +add_to_extratags:ForumFic + +# true, false, threadmarksonly +add_chapter_dates:false + +strip_chapter_numbers:false + +add_to_extra_valid_entries:,titletags +# '.NOREPL' tells the system to *not* apply title's +# in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_titletags:title.NOREPL + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:titletags + +include_metadata_pre: +# only keep titletags with ( or [ in. + titletags=~[\[\(] + +replace_metadata: +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> +# remove anything outside () or [] + titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + titletags=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + titletags=> */ *=>, + titletags=> x =>, +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> + +extra_titlepage_entries: titletags + +## '.SPLIT' teels the system to split by ',' +add_to_include_subject_tags:,titletags.SPLIT + +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +description_limit:1500 + [grangerenchanted.com] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In diff --git a/fanficfare/story.py b/fanficfare/story.py index 19c30ef..2e7c2ae 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -465,7 +465,7 @@ class Story(Configurable): if key == 'dateUpdated' and value: # Last Update tags for Bill. - self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m")) + self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True) self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d")) @@ -757,8 +757,12 @@ class Story(Configurable): # includelist prevents infinite recursion of include_in_'s if self.hasConfig("include_in_"+listname) and listname not in includelist: for k in self.getConfigList("include_in_"+listname): + ldorepl = doreplacements + if k.endswith('.NOREPL'): + k = k[:-len('.NOREPL')] + ldorepl = False retlist.extend(self.getList(k,removeallentities=False, - doreplacements=doreplacements,includelist=includelist+[listname])) + doreplacements=ldorepl,includelist=includelist+[listname])) else: if not self.isList(listname): @@ -813,7 +817,16 @@ class Story(Configurable): # metadata all go into dc:subject tags, but only if they are configured. for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems(): - if name in tags_list: + if name+'.SPLIT' in tags_list: + flist=[] + if isinstance(value,list): + for tag in value: + flist.extend(tag.split(',')) + else: + flist.extend(value) + for tag in flist: + subjectset.add(tag) + elif name in tags_list: if isinstance(value,list): for tag in value: subjectset.add(tag) @@ -911,6 +924,7 @@ class Story(Configurable): #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl)) # apply coverexclusion to explicit covers, too. Primarily for ffnet imageu. + #print("[[[[[\n\n %s %s \n\n]]]]]]]"%(imgurl,coverexclusion)) if cover and coverexclusion and re.search(coverexclusion,imgurl): return (None,None) From ddc3607df609d2a556b5a39fbff1e64c7659c210 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 25 Jun 2015 18:19:38 -0500 Subject: [PATCH 11/32] forum adapters - Take pub/update dates from index post if not from threadmarks. --- .../adapters/adapter_forumsspacebattlescom.py | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index 5351398..58d89c9 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -128,18 +128,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') if len(markas) > 1: for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]: - datestr=None - datetag = atag.find_next_sibling('div',{'class':'extra'}).find('span',{'class':'DateTime'}) - if datetag: - datestr = datetag['title'] - else: - datetag = atag.find_next_sibling('div',{'class':'extra'}).find('abbr',{'class':'DateTime'}) - if datetag: - datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring']) - # Apr 24, 2015 at 4:39 AM - # May 1, 2015 at 5:47 AM - datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours. - date = makeDate(datestr, self.dateformat) + date = self.make_date(atag.find_next_sibling('div',{'class':'extra'})) if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'): self.story.setMetadata('datePublished', date) if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'): @@ -149,6 +138,8 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): name = '%s %s'%(name,date) self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) + + soup = soup.find('li') # limit first post for date stuff below. ('#' posts above) # Now go hunting for the 'chapter list'. bq = soup.find('blockquote') # assume first posting contains TOC urls. @@ -177,9 +168,39 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): # remove "First Post" if included in list. logger.debug("delete dup chapter: %s %s"%self.chapterUrls[0]) del self.chapterUrls[0] - + + # Didn't use threadmarks, so take created/updated dates + # from the 'first' posting created and updated. + date = self.make_date(soup.find('a',{'class':'datePermalink'})) + if date: + self.story.setMetadata('datePublished', date) + self.story.setMetadata('dateUpdated', date) # updated overwritten below if found. + + date = self.make_date(soup.find('div',{'class':'editDate'})) + if date: + self.story.setMetadata('dateUpdated', date) + self.story.setMetadata('numChapters',len(self.chapterUrls)) + def make_date(self,parenttag): # forums use a BS thing where dates + # can appear different if recent. + datestr=None + try: + datetag = parenttag.find('span',{'class':'DateTime'}) + if datetag: + datestr = datetag['title'] + else: + datetag = parenttag.find('abbr',{'class':'DateTime'}) + if datetag: + datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring']) + # Apr 24, 2015 at 4:39 AM + # May 1, 2015 at 5:47 AM + datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours. + return makeDate(datestr, self.dateformat) + except: + logger.debug('No date found in %s'%parenttag) + return None + # grab the text for an individual chapter. def getChapterText(self, url): logger.debug('Getting chapter text from: %s' % url) From dfae1046742928f9035496cea1dd5ad1f3d78312 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 26 Jun 2015 12:18:55 -0500 Subject: [PATCH 12/32] forum adapters - Fixes --- calibre-plugin/plugin-defaults.ini | 12 ++++++------ fanficfare/adapters/adapter_forumsspacebattlescom.py | 5 +---- fanficfare/defaults.ini | 12 ++++++------ 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index e8e27a3..5a953a6 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -1045,7 +1045,7 @@ cover_exclusion_regexp:/res/css/bir.png cover_exclusion_regexp:/clear.png -add_to_extratags:ForumFic +add_to_extratags:,ForumFic strip_chapter_numbers:false @@ -1081,19 +1081,19 @@ replace_metadata: extra_titlepage_entries: titletags -## '.SPLIT' teels the system to split by ',' +## '.SPLIT' tells the system to split by ',' add_to_include_subject_tags:,titletags.SPLIT datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S -description_limit:1500 +description_limit:500 [forums.spacebattles.com] cover_exclusion_regexp:/clear.png -add_to_extratags:ForumFic +add_to_extratags:,ForumFic # true, false, threadmarksonly add_chapter_dates:false @@ -1129,13 +1129,13 @@ replace_metadata: extra_titlepage_entries: titletags -## '.SPLIT' teels the system to split by ',' +## '.SPLIT' tells the system to split by ',' add_to_include_subject_tags:,titletags.SPLIT datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S -description_limit:1500 +description_limit:500 [grangerenchanted.com] ## Some sites require login (or login for some rated stories) The diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index 58d89c9..43fd525 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -133,13 +133,10 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): self.story.setMetadata('datePublished', date) if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'): self.story.setMetadata('dateUpdated', date) - - if self.getConfig('add_chapter_dates') in ['true','threadmarksonly']: - name = '%s %s'%(name,date) self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) - soup = soup.find('li') # limit first post for date stuff below. ('#' posts above) + soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above) # Now go hunting for the 'chapter list'. bq = soup.find('blockquote') # assume first posting contains TOC urls. diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 52256f8..07d62ba 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -1031,7 +1031,7 @@ cover_exclusion_regexp:/res/css/bir.png cover_exclusion_regexp:/clear.png -add_to_extratags:ForumFic +add_to_extratags:,ForumFic strip_chapter_numbers:false @@ -1067,19 +1067,19 @@ replace_metadata: extra_titlepage_entries: titletags -## '.SPLIT' teels the system to split by ',' +## '.SPLIT' tells the system to split by ',' add_to_include_subject_tags:,titletags.SPLIT datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S -description_limit:1500 +description_limit:500 [forums.spacebattles.com] cover_exclusion_regexp:/clear.png -add_to_extratags:ForumFic +add_to_extratags:,ForumFic # true, false, threadmarksonly add_chapter_dates:false @@ -1115,13 +1115,13 @@ replace_metadata: extra_titlepage_entries: titletags -## '.SPLIT' teels the system to split by ',' +## '.SPLIT' tells the system to split by ',' add_to_include_subject_tags:,titletags.SPLIT datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S -description_limit:1500 +description_limit:500 [grangerenchanted.com] ## Some sites require login (or login for some rated stories) The From 1de8755d36ebd598a0a5d66e91e8940d31470c91 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 29 Jun 2015 10:28:53 -0500 Subject: [PATCH 13/32] Add more forums tweaks and '(new)' chapter mark option. --- calibre-plugin/plugin-defaults.ini | 22 +++++-------------- .../adapters/adapter_forumsspacebattlescom.py | 8 ++++++- fanficfare/adapters/base_adapter.py | 6 ++++- fanficfare/story.py | 4 +++- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index 5a953a6..d542eed 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -1045,13 +1045,8 @@ cover_exclusion_regexp:/res/css/bir.png cover_exclusion_regexp:/clear.png -add_to_extratags:,ForumFic - strip_chapter_numbers:false -# true, false, threadmarksonly -add_chapter_dates:false - add_to_extra_valid_entries:,titletags # '.NOREPL' tells the system to *not* apply title's # in/exclude/replace_metadata -- Only works on include_in_ lines. @@ -1066,9 +1061,6 @@ include_metadata_pre: titletags=~[\[\(] replace_metadata: -# remove 'Thread' and the next word, usually "Thread 2", "Thread -# four", "Thread iv", etc - title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> # remove anything outside () or [] titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 # remove () [] @@ -1078,6 +1070,9 @@ replace_metadata: titletags=> x =>, # remove [] or () blocks and leading/trailing spaces title=> *[\(\[]([^\]\)]+)[\)\]] *=> +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> extra_titlepage_entries: titletags @@ -1093,11 +1088,6 @@ description_limit:500 cover_exclusion_regexp:/clear.png -add_to_extratags:,ForumFic - -# true, false, threadmarksonly -add_chapter_dates:false - strip_chapter_numbers:false add_to_extra_valid_entries:,titletags @@ -1114,9 +1104,6 @@ include_metadata_pre: titletags=~[\[\(] replace_metadata: -# remove 'Thread' and the next word, usually "Thread 2", "Thread -# four", "Thread iv", etc - title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> # remove anything outside () or [] titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 # remove () [] @@ -1126,6 +1113,9 @@ replace_metadata: titletags=> x =>, # remove [] or () blocks and leading/trailing spaces title=> *[\(\[]([^\]\)]+)[\)\]] *=> +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> extra_titlepage_entries: titletags diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index 43fd525..839c2cd 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -146,6 +146,9 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): for iframe in bq.find_all('iframe'): iframe.extract() # calibre book reader & editor don't like iframes to youtube. + for qdiv in bq.find_all('div',{'class':'quoteExpand'}): + qdiv.extract() # Remove
click to expand
+ self.setDescription(useurl,bq) # otherwise, use first post links--include first post since that's @@ -163,7 +166,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): self.chapterUrls.append((name,url)) if url == useurl and 'First Post' == self.chapterUrls[0][0]: # remove "First Post" if included in list. - logger.debug("delete dup chapter: %s %s"%self.chapterUrls[0]) + logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0]) del self.chapterUrls[0] # Didn't use threadmarks, so take created/updated dates @@ -218,4 +221,7 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): for iframe in bq.find_all('iframe'): iframe.extract() # calibre book reader & editor don't like iframes to youtube. + for qdiv in bq.find_all('div',{'class':'quoteExpand'}): + qdiv.extract() # Remove
click to expand
+ return self.utf8FromSoup(url,bq) diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index a0a0b7f..efc774c 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -361,6 +361,7 @@ class BaseSiteAdapter(Configurable): self.getStoryMetadataOnly(get_cover=True) for index, (title,url) in enumerate(self.chapterUrls): + marknewchap = False if (self.chapterFirst!=None and index < self.chapterFirst) or \ (self.chapterLast!=None and index > self.chapterLast): self.story.addChapter(url, @@ -379,9 +380,12 @@ class BaseSiteAdapter(Configurable): partial(cachedfetch,self._fetchUrlRaw,self.oldimgs)) if not data: data = self.getChapterText(url) + # if configured and has existing chapters + marknewchap = (self.getConfig('mark_new_chapters')=='true' and self.oldchapters or self.oldchaptersmap) self.story.addChapter(url, removeEntities(title), - removeEntities(data)) + removeEntities(data), + marknewchap) self.storyDone = True # include image, but no cover from story, add default_cover_image cover. diff --git a/fanficfare/story.py b/fanficfare/story.py index 2e7c2ae..b9abdd8 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -840,10 +840,12 @@ class Story(Configurable): return list(subjectset | set(self.getConfigList("extratags"))) - def addChapter(self, url, title, html): + def addChapter(self, url, title, html, marknewchap=False): if self.getConfig('strip_chapter_numbers') and \ self.getConfig('chapter_title_strip_pattern'): title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title) + if marknewchap: + title=u'(new) %s'%title self.chapters.append( (url,title,html) ) def getChapters(self,fortoc=False): From 4dcfd6e4bec0744048c24991fa1551852a05fc59 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 29 Jun 2015 13:07:21 -0500 Subject: [PATCH 14/32] Add [base_efiction] and [base_xenforoforum] sections, add base_xenforoforum_adapter, document new options. --- calibre-plugin/config.py | 4 +- calibre-plugin/fff_util.py | 8 +- calibre-plugin/plugin-defaults.ini | 155 +++++++-------- fanficfare/adapters/__init__.py | 22 ++- .../adapters/adapter_forumsspacebattlescom.py | 185 +----------------- .../adapter_forumssufficientvelocitycom.py | 7 +- fanficfare/adapters/base_adapter.py | 5 + fanficfare/adapters/base_efiction_adapter.py | 5 + fanficfare/cli.py | 2 +- fanficfare/configurable.py | 20 +- fanficfare/defaults.ini | 165 +++++++--------- fanficfare/story.py | 2 +- webservice/main.py | 2 +- 13 files changed, 203 insertions(+), 379 deletions(-) diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index 937c2a5..1686d07 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -89,7 +89,7 @@ from calibre_plugins.fanficfare_plugin.dialogs \ EditTextDialog, IniTextDialog, RejectUrlEntry) from calibre_plugins.fanficfare_plugin.fanficfare.adapters \ - import getConfigSections + import getSiteSections from calibre_plugins.fanficfare_plugin.common_utils \ import ( KeyboardConfigDialog, PrefsViewerDialog ) @@ -922,7 +922,7 @@ class CalibreCoverTab(QWidget): self.gc_dropdowns = {} - sitelist = getConfigSections() + sitelist = getSiteSections() sitelist.sort() sitelist.insert(0,_("Default")) for site in sitelist: diff --git a/calibre-plugin/fff_util.py b/calibre-plugin/fff_util.py index b1e8a4d..0ba7227 100644 --- a/calibre-plugin/fff_util.py +++ b/calibre-plugin/fff_util.py @@ -23,12 +23,12 @@ def get_fff_personalini(): def get_fff_config(url,fileform="epub",personalini=None): if not personalini: personalini = get_fff_personalini() - site='unknown' + sections=['unknown'] try: - site = adapters.getConfigSectionFor(url) + sections = adapters.getConfigSectionsFor(url) except Exception as e: - logger.debug("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site)) - configuration = Configuration(site,fileform) + logger.debug("Failed trying to get ini config for url(%s): %s, using section %s instead"%(url,e,sections)) + configuration = Configuration(sections,fileform) configuration.readfp(StringIO(get_resources("plugin-defaults.ini"))) configuration.readfp(StringIO(personalini)) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index d542eed..329eac5 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -299,6 +299,13 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+ ## "The Beginning" => "1. The Beginning" chapter_title_add_pattern:${index}. ${title} +## If true, when updating an epub that already has old chapters, new +## chapters will be marked in the TOC and chapter header by prepending +## '(new) ' to the chapter title. So 'The Big Fight' will become +## '4. (new) The Big Fight' if both mark_new_chapters and +## add_chapter_numbers are set true. +mark_new_chapters:false + ## Uses a python template substitution. The ${title} is the default ## title of a new anthology, in the case of a series, or ## the first book title otherwise. This is only applied to new @@ -330,12 +337,74 @@ sort_ships:false ## User-agent user_agent:FFF/2.X +## Added for [base_xenforoforum], but can be used with other sites, +## too. Limit the 'description' to the first X *characters* +## collected. Character count includes HTML tags, so it can be +## non-intuitive. +#description_limit:1000 + +[base_efiction] ## At the time of writing, eFiction Base adapters allow downloading ## the whole story in bulk using the 'Print' feature. If 'bulk_load' ## is set to 'true', both metadata and chapters can be loaded in one ## step bulk_load:true +[base_xenforoforum] +## Currently only forums.spacebattles.com and forums.sufficientvelocity.com + +cover_exclusion_regexp:/clear.png + +## I saw lots of chapters name simply '1.1' etc during testing. +strip_chapter_numbers:false + +## Copy title to tagsfromtitle for parsing tags. +add_to_extra_valid_entries:,tagsfromtitle + +## '.NOREPL' tells the system to *not* apply title's +## in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_tagsfromtitle:title.NOREPL + +tagsfromtitle_label:Tags from Title + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:tagsfromtitle + +include_metadata_pre: +# only keep tagsfromtitle with ( or [ in. + tagsfromtitle=~[\[\(] + +replace_metadata: +# remove anything outside () or [] + tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + tagsfromtitle=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + tagsfromtitle=> */ *=>, + tagsfromtitle=> x =>, + +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> + +extra_titlepage_entries: tagsfromtitle + +## '.SPLIT' tells the system to split by ',' +add_to_include_subject_tags:,tagsfromtitle.SPLIT + +## base_xenforoforum reads Published and Updated datetimes from +## Threadmarks if used, or from the posted & updated times of the +## 'first' post if no threadmarks. +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +## Only take the first X characters of the 'first' post to use as +## the description. +description_limit:500 + ## Each output format has a section that overrides [defaults] [html] @@ -1041,91 +1110,11 @@ extra_valid_entries:size # don't show twitter icon. cover_exclusion_regexp:/res/css/bir.png -[forums.sufficientvelocity.com] - -cover_exclusion_regexp:/clear.png - -strip_chapter_numbers:false - -add_to_extra_valid_entries:,titletags -# '.NOREPL' tells the system to *not* apply title's -# in/exclude/replace_metadata -- Only works on include_in_ lines. -include_in_titletags:title.NOREPL - -## might want to do this, maybe not. Will often include category, but -## also often include non-category stuff. -# include_in_category:titletags - -include_metadata_pre: -# only keep titletags with ( or [ in. - titletags=~[\[\(] - -replace_metadata: -# remove anything outside () or [] - titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 -# remove () [] - titletags=>[\(\)\[\]]=> -# change (spaces)slash(spaces) to comma - titletags=> */ *=>, - titletags=> x =>, -# remove [] or () blocks and leading/trailing spaces - title=> *[\(\[]([^\]\)]+)[\)\]] *=> -# remove 'Thread' and the next word, usually "Thread 2", "Thread -# four", "Thread iv", etc - title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> - -extra_titlepage_entries: titletags - -## '.SPLIT' tells the system to split by ',' -add_to_include_subject_tags:,titletags.SPLIT - -datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S -dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S - -description_limit:500 - [forums.spacebattles.com] +## see [base_xenforoforum] -cover_exclusion_regexp:/clear.png - -strip_chapter_numbers:false - -add_to_extra_valid_entries:,titletags -# '.NOREPL' tells the system to *not* apply title's -# in/exclude/replace_metadata -- Only works on include_in_ lines. -include_in_titletags:title.NOREPL - -## might want to do this, maybe not. Will often include category, but -## also often include non-category stuff. -# include_in_category:titletags - -include_metadata_pre: -# only keep titletags with ( or [ in. - titletags=~[\[\(] - -replace_metadata: -# remove anything outside () or [] - titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 -# remove () [] - titletags=>[\(\)\[\]]=> -# change (spaces)slash(spaces) to comma - titletags=> */ *=>, - titletags=> x =>, -# remove [] or () blocks and leading/trailing spaces - title=> *[\(\[]([^\]\)]+)[\)\]] *=> -# remove 'Thread' and the next word, usually "Thread 2", "Thread -# four", "Thread iv", etc - title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> - -extra_titlepage_entries: titletags - -## '.SPLIT' tells the system to split by ',' -add_to_include_subject_tags:,titletags.SPLIT - -datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S -dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S - -description_limit:500 +[forums.sufficientvelocity.com] +## see [base_xenforoforum] [grangerenchanted.com] ## Some sites require login (or login for some rated stories) The diff --git a/fanficfare/adapters/__init__.py b/fanficfare/adapters/__init__.py index 301cd9c..15110b6 100644 --- a/fanficfare/adapters/__init__.py +++ b/fanficfare/adapters/__init__.py @@ -196,14 +196,24 @@ def getAdapter(config,url,anyurl=False): # No adapter found. raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] ) -def getConfigSections(): +def getSiteSections(): + # doesn't include base sections. Sections rather than site DNS because of squidge/peja return [cls.getConfigSection() for cls in __class_list] +def getConfigSections(): + # does include base sections. + sections = set() + for cls in __class_list: + sections.update(cls.getConfigSections()) + return sections + def get_bulk_load_sites(): # for now, all eFiction Base adapters are assumed to allow bulk_load. - return [cls.getConfigSection().replace('www.','') for cls in - filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter), - __class_list)] + sections = set() + for cls in filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter), + __class_list): + sections.update( [ x.replace('www.','') for x in cls.getConfigSections() ] ) + return sections def getSiteExamples(): l=[] @@ -211,10 +221,10 @@ def getSiteExamples(): l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split())) return l -def getConfigSectionFor(url): +def getConfigSectionsFor(url): (cls,fixedurl) = getClassFor(url) if cls: - return cls.getConfigSection() + return cls.getConfigSections() # No adapter found. raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] ) diff --git a/fanficfare/adapters/adapter_forumsspacebattlescom.py b/fanficfare/adapters/adapter_forumsspacebattlescom.py index 839c2cd..0a58c65 100644 --- a/fanficfare/adapters/adapter_forumsspacebattlescom.py +++ b/fanficfare/adapters/adapter_forumsspacebattlescom.py @@ -24,47 +24,18 @@ import urllib2 from ..htmlcleanup import stripHTML from .. import exceptions as exceptions -from base_adapter import BaseSiteAdapter, makeDate +from base_xenforoforum_adapter import BaseXenForoForumAdapter def getClass(): return ForumsSpacebattlesComAdapter -logger = logging.getLogger(__name__) - -class ForumsSpacebattlesComAdapter(BaseSiteAdapter): +class ForumsSpacebattlesComAdapter(BaseXenForoForumAdapter): def __init__(self, config, url): - BaseSiteAdapter.__init__(self, config, url) - - self.decode = ["utf8", - "Windows-1252"] # 1252 is a superset of iso-8859-1. - # Most sites that claim to be - # iso-8859-1 (and some that claim to be - # utf8) are really windows-1252. - - - # get storyId from url--url validation guarantees query is only sid=1234 - self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2]) - - - # get storyId from url--url validation guarantees query correct - m = re.match(self.getSiteURLPattern(),url) - if m: - self.story.setMetadata('storyId',m.group('id')) - - # normalized story URL. - self._setURL(self.getURLPrefix() + '/'+m.group('tp')+'/'+self.story.getMetadata('storyId')+'/') - else: - raise exceptions.InvalidStoryURL(url, - self.getSiteDomain(), - self.getSiteExampleURLs()) + BaseXenForoForumAdapter.__init__(self, config, url) # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','fsb') - - # The date format will vary from site to site. - # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - self.dateformat = "%b %d, %Y at %I:%M %p" @staticmethod # must be @staticmethod, don't remove it. def getSiteDomain(): @@ -73,155 +44,5 @@ class ForumsSpacebattlesComAdapter(BaseSiteAdapter): @classmethod def getURLPrefix(cls): - # The site domain. Does have www here, if it uses it. return 'https://' + cls.getSiteDomain() - @classmethod - def getSiteExampleURLs(cls): - return cls.getURLPrefix()+"/threads/some-story-name.123456/" - - def getSiteURLPattern(self): - return r"https?://"+re.escape(self.getSiteDomain())+r"/(?Pthreads|posts)/(.+\.)?(?P\d+)/" - - def use_pagecache(self): - ''' - adapters that will work with the page cache need to implement - this and change it to True. - ''' - return True - - ## Getting the chapter list and the meta data, plus 'is adult' checking. - def extractChapterUrlsAndMetadata(self): - - useurl = self.url - logger.info("url: "+useurl) - - try: - (data,opened) = self._fetchUrlOpened(useurl) - useurl = opened.geturl() - logger.info("use useurl: "+useurl) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - - # use BeautifulSoup HTML parser to make everything easier to find. - soup = self.make_soup(data) - - a = soup.find('h3',{'class':'userText'}).find('a') - self.story.addToList('authorId',a['href'].split('/')[1]) - self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href']) - self.story.addToList('author',a.text) - - h1 = soup.find('div',{'class':'titleBar'}).h1 - self.story.setMetadata('title',stripHTML(h1)) - - if '#' in useurl: - anchorid = useurl.split('#')[1] - soup = soup.find('li',id=anchorid) - else: - # try threadmarks if no '#' in , require at least 2. - threadmarksa = soup.find('a',{'class':'threadmarksTrigger'}) - if threadmarksa: - soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href'])) - markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') - if len(markas) > 1: - for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]: - date = self.make_date(atag.find_next_sibling('div',{'class':'extra'})) - if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'): - self.story.setMetadata('datePublished', date) - if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'): - self.story.setMetadata('dateUpdated', date) - - self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) - - soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above) - - # Now go hunting for the 'chapter list'. - bq = soup.find('blockquote') # assume first posting contains TOC urls. - - bq.name='div' - - for iframe in bq.find_all('iframe'): - iframe.extract() # calibre book reader & editor don't like iframes to youtube. - - for qdiv in bq.find_all('div',{'class':'quoteExpand'}): - qdiv.extract() # Remove
click to expand
- - self.setDescription(useurl,bq) - - # otherwise, use first post links--include first post since that's - if not self.chapterUrls: - self.chapterUrls.append(("First Post",useurl)) - for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]: - logger.debug("found chapurl:%s"%url) - if not url.startswith('http'): - url = self.getURLPrefix()+'/'+url - - if (url.startswith(self.getURLPrefix()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): - # brute force way to deal with SB's http->https change when hardcoded http urls. - url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix()) - logger.debug("used chapurl:%s"%(url)) - self.chapterUrls.append((name,url)) - if url == useurl and 'First Post' == self.chapterUrls[0][0]: - # remove "First Post" if included in list. - logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0]) - del self.chapterUrls[0] - - # Didn't use threadmarks, so take created/updated dates - # from the 'first' posting created and updated. - date = self.make_date(soup.find('a',{'class':'datePermalink'})) - if date: - self.story.setMetadata('datePublished', date) - self.story.setMetadata('dateUpdated', date) # updated overwritten below if found. - - date = self.make_date(soup.find('div',{'class':'editDate'})) - if date: - self.story.setMetadata('dateUpdated', date) - - self.story.setMetadata('numChapters',len(self.chapterUrls)) - - def make_date(self,parenttag): # forums use a BS thing where dates - # can appear different if recent. - datestr=None - try: - datetag = parenttag.find('span',{'class':'DateTime'}) - if datetag: - datestr = datetag['title'] - else: - datetag = parenttag.find('abbr',{'class':'DateTime'}) - if datetag: - datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring']) - # Apr 24, 2015 at 4:39 AM - # May 1, 2015 at 5:47 AM - datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours. - return makeDate(datestr, self.dateformat) - except: - logger.debug('No date found in %s'%parenttag) - return None - - # grab the text for an individual chapter. - def getChapterText(self, url): - logger.debug('Getting chapter text from: %s' % url) - - (data,opened) = self._fetchUrlOpened(url) - url = opened.geturl() - logger.debug("chapter URL redirected to: %s"%url) - - soup = self.make_soup(data) - - if '#' in url: - anchorid = url.split('#')[1] - soup = soup.find('li',id=anchorid) - bq = soup.find('blockquote') - - bq.name='div' - - for iframe in bq.find_all('iframe'): - iframe.extract() # calibre book reader & editor don't like iframes to youtube. - - for qdiv in bq.find_all('div',{'class':'quoteExpand'}): - qdiv.extract() # Remove
click to expand
- - return self.utf8FromSoup(url,bq) diff --git a/fanficfare/adapters/adapter_forumssufficientvelocitycom.py b/fanficfare/adapters/adapter_forumssufficientvelocitycom.py index f16785e..883dc74 100644 --- a/fanficfare/adapters/adapter_forumssufficientvelocitycom.py +++ b/fanficfare/adapters/adapter_forumssufficientvelocitycom.py @@ -15,15 +15,15 @@ # limitations under the License. # -from adapter_forumsspacebattlescom import ForumsSpacebattlesComAdapter +from base_xenforoforum_adapter import BaseXenForoForumAdapter def getClass(): return ForumsSufficientVelocityComAdapter -class ForumsSufficientVelocityComAdapter(ForumsSpacebattlesComAdapter): +class ForumsSufficientVelocityComAdapter(BaseXenForoForumAdapter): def __init__(self, config, url): - ForumsSpacebattlesComAdapter.__init__(self, config, url) + BaseXenForoForumAdapter.__init__(self, config, url) # Each adapter needs to have a unique site abbreviation. self.story.setMetadata('siteabbrev','fsv') @@ -35,5 +35,4 @@ class ForumsSufficientVelocityComAdapter(ForumsSpacebattlesComAdapter): @classmethod def getURLPrefix(cls): - # The site domain. Does have www here, if it uses it. return 'http://' + cls.getSiteDomain() diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index efc774c..aebf330 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -452,6 +452,11 @@ class BaseSiteAdapter(Configurable): "Only needs to be overriden if != site domain." return cls.getSiteDomain() + @classmethod + def getConfigSections(cls): + "Only needs to be overriden if has additional ini sections." + return [cls.getConfigSection()] + @classmethod def stripURLParameters(cls,url): "Only needs to be overriden if URL contains more than one parameter" diff --git a/fanficfare/adapters/base_efiction_adapter.py b/fanficfare/adapters/base_efiction_adapter.py index 354072f..7d0fa23 100644 --- a/fanficfare/adapters/base_efiction_adapter.py +++ b/fanficfare/adapters/base_efiction_adapter.py @@ -70,6 +70,11 @@ class BaseEfictionAdapter(BaseSiteAdapter): self.triedAcceptWarnings = False self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + @classmethod + def getConfigSections(cls): + "Only needs to be overriden if has additional ini sections." + return ['base_efiction',cls.getConfigSection()] + @classmethod def getAcceptDomains(cls): return [cls.getSiteDomain(),'www.' + cls.getSiteDomain()] diff --git a/fanficfare/cli.py b/fanficfare/cli.py index 0856ce7..32ae102 100644 --- a/fanficfare/cli.py +++ b/fanficfare/cli.py @@ -184,7 +184,7 @@ def do_download(arg, url = arg try: - configuration = Configuration(adapters.getConfigSectionFor(url), options.format) + configuration = Configuration(adapters.getConfigSectionsFor(url), options.format) except exceptions.UnknownSite, e: if options.list or options.normalize: # list for page doesn't have to be a supported site. diff --git a/fanficfare/configurable.py b/fanficfare/configurable.py index e6f6ee8..f38f474 100644 --- a/fanficfare/configurable.py +++ b/fanficfare/configurable.py @@ -77,10 +77,12 @@ formatsections = ['html','txt','epub','mobi'] othersections = ['defaults','overrides'] def get_valid_sections(): - sites = adapters.getConfigSections() + sites = adapters.getConfigSections() sitesections = list(othersections) for section in sites: sitesections.append(section) + # also allows [www.base_efiction] and [www.base_forum]. Not + # likely to matter. if section.startswith('www.'): # add w/o www if has www sitesections.append(section[4:]) @@ -130,6 +132,7 @@ def get_valid_set_options(): 'replace_hr':(None,None,boollist), 'sort_ships':(None,None,boollist), 'strip_chapter_numbers':(None,None,boollist), + 'mark_new_chapters':(None,None,boollist), 'titlepage_use_table':(None,None,boollist), 'use_ssl_unverified_context':(None,None,boollist), @@ -212,6 +215,7 @@ def get_valid_keywords(): 'chapter_start', 'chapter_title_add_pattern', 'chapter_title_strip_pattern', + 'mark_new_chapters', 'check_next_chapter', 'skip_author_cover', 'collect_series', @@ -224,6 +228,7 @@ def get_valid_keywords(): 'datePublished_format', 'dateUpdated_format', 'default_cover_image', + 'description_limit', 'do_update_hook', 'exclude_notes', 'extra_logpage_entries', @@ -331,7 +336,8 @@ def make_generate_cover_settings(param): class Configuration(ConfigParser.SafeConfigParser): - def __init__(self, site, fileform): + def __init__(self, sections, fileform): + site = sections[-1] # first section is site DN. ConfigParser.SafeConfigParser.__init__(self) self.linenos=dict() # key by section or section,key -> lineno @@ -339,6 +345,11 @@ class Configuration(ConfigParser.SafeConfigParser): ## [injected] section has even less priority than [defaults] self.sectionslist = ['defaults','injected'] + ## add other sections (not including site DN) after defaults, + ## but before site-specific. + for section in sections[:-1]: + self.addConfigSection(section) + if site.startswith("www."): sitewith = site sitewithout = site.replace("www.","") @@ -348,8 +359,13 @@ class Configuration(ConfigParser.SafeConfigParser): self.addConfigSection(sitewith) self.addConfigSection(sitewithout) + if fileform: self.addConfigSection(fileform) + ## add other sections:fileform (not including site DN) + ## after fileform, but before site-specific:fileform. + for section in sections[:-1]: + self.addConfigSection(section+":"+fileform) self.addConfigSection(sitewith+":"+fileform) self.addConfigSection(sitewithout+":"+fileform) self.addConfigSection("overrides") diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 07d62ba..26de893 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -306,6 +306,13 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+ ## "The Beginning" => "1. The Beginning" chapter_title_add_pattern:${index}. ${title} +## If true, when updating an epub that already has old chapters, new +## chapters will be marked in the TOC and chapter header by prepending +## '(new) ' to the chapter title. So 'The Big Fight' will become +## '4. (new) The Big Fight' if both mark_new_chapters and +## add_chapter_numbers are set true. +mark_new_chapters:false + ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates ## on '/', so use replace_metadata to change separator first if ## needed. Something like: ships=>[ ]*(/|&|&)[ ]*=>/ You can use @@ -327,12 +334,74 @@ sort_ships:false ## User-agent user_agent:FFF/2.X +## Added for [base_xenforoforum], but can be used with other sites, +## too. Limit the 'description' to the first X *characters* +## collected. Character count includes HTML tags, so it can be +## non-intuitive. +#description_limit:1000 + +[base_efiction] ## At the time of writing, eFiction Base adapters allow downloading ## the whole story in bulk using the 'Print' feature. If 'bulk_load' ## is set to 'true', both metadata and chapters can be loaded in one ## step bulk_load:true +[base_xenforoforum] +## Currently only forums.spacebattles.com and forums.sufficientvelocity.com + +cover_exclusion_regexp:/clear.png + +## I saw lots of chapters name simply '1.1' etc during testing. +strip_chapter_numbers:false + +## Copy title to tagsfromtitle for parsing tags. +add_to_extra_valid_entries:,tagsfromtitle + +## '.NOREPL' tells the system to *not* apply title's +## in/exclude/replace_metadata -- Only works on include_in_ lines. +include_in_tagsfromtitle:title.NOREPL + +tagsfromtitle_label:Tags from Title + +## might want to do this, maybe not. Will often include category, but +## also often include non-category stuff. +# include_in_category:tagsfromtitle + +include_metadata_pre: +# only keep tagsfromtitle with ( or [ in. + tagsfromtitle=~[\[\(] + +replace_metadata: +# remove anything outside () or [] + tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 +# remove () [] + tagsfromtitle=>[\(\)\[\]]=> +# change (spaces)slash(spaces) to comma + tagsfromtitle=> */ *=>, + tagsfromtitle=> x =>, + +# remove [] or () blocks and leading/trailing spaces + title=> *[\(\[]([^\]\)]+)[\)\]] *=> +# remove 'Thread' and the next word, usually "Thread 2", "Thread +# four", "Thread iv", etc + title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> + +extra_titlepage_entries: tagsfromtitle + +## '.SPLIT' tells the system to split by ',' +add_to_include_subject_tags:,tagsfromtitle.SPLIT + +## base_xenforoforum reads Published and Updated datetimes from +## Threadmarks if used, or from the posted & updated times of the +## 'first' post if no threadmarks. +datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S +dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S + +## Only take the first X characters of the 'first' post to use as +## the description. +description_limit:500 + ## Each output format has a section that overrides [defaults] [html] @@ -1027,101 +1096,11 @@ extra_valid_entries:size # don't show twitter icon. cover_exclusion_regexp:/res/css/bir.png -[forums.sufficientvelocity.com] - -cover_exclusion_regexp:/clear.png - -add_to_extratags:,ForumFic - -strip_chapter_numbers:false - -# true, false, threadmarksonly -add_chapter_dates:false - -add_to_extra_valid_entries:,titletags -# '.NOREPL' tells the system to *not* apply title's -# in/exclude/replace_metadata -- Only works on include_in_ lines. -include_in_titletags:title.NOREPL - -## might want to do this, maybe not. Will often include category, but -## also often include non-category stuff. -# include_in_category:titletags - -include_metadata_pre: -# only keep titletags with ( or [ in. - titletags=~[\[\(] - -replace_metadata: -# remove 'Thread' and the next word, usually "Thread 2", "Thread -# four", "Thread iv", etc - title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> -# remove anything outside () or [] - titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 -# remove () [] - titletags=>[\(\)\[\]]=> -# change (spaces)slash(spaces) to comma - titletags=> */ *=>, - titletags=> x =>, -# remove [] or () blocks and leading/trailing spaces - title=> *[\(\[]([^\]\)]+)[\)\]] *=> - -extra_titlepage_entries: titletags - -## '.SPLIT' tells the system to split by ',' -add_to_include_subject_tags:,titletags.SPLIT - -datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S -dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S - -description_limit:500 - [forums.spacebattles.com] +## see [base_xenforoforum] -cover_exclusion_regexp:/clear.png - -add_to_extratags:,ForumFic - -# true, false, threadmarksonly -add_chapter_dates:false - -strip_chapter_numbers:false - -add_to_extra_valid_entries:,titletags -# '.NOREPL' tells the system to *not* apply title's -# in/exclude/replace_metadata -- Only works on include_in_ lines. -include_in_titletags:title.NOREPL - -## might want to do this, maybe not. Will often include category, but -## also often include non-category stuff. -# include_in_category:titletags - -include_metadata_pre: -# only keep titletags with ( or [ in. - titletags=~[\[\(] - -replace_metadata: -# remove 'Thread' and the next word, usually "Thread 2", "Thread -# four", "Thread iv", etc - title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> -# remove anything outside () or [] - titletags=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 -# remove () [] - titletags=>[\(\)\[\]]=> -# change (spaces)slash(spaces) to comma - titletags=> */ *=>, - titletags=> x =>, -# remove [] or () blocks and leading/trailing spaces - title=> *[\(\[]([^\]\)]+)[\)\]] *=> - -extra_titlepage_entries: titletags - -## '.SPLIT' tells the system to split by ',' -add_to_include_subject_tags:,titletags.SPLIT - -datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S -dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S - -description_limit:500 +[forums.sufficientvelocity.com] +## see [base_xenforoforum] [grangerenchanted.com] ## Some sites require login (or login for some rated stories) The diff --git a/fanficfare/story.py b/fanficfare/story.py index b9abdd8..dd578ce 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -412,7 +412,7 @@ class Story(Configurable): except: self.metadata = {'version':'4.4'} self.in_ex_cludes = {} - self.chapters = [] # chapters will be tuples of (title,html) + self.chapters = [] # chapters will be tuples of (url,title,html) self.imgurls = [] self.imgtuples = [] diff --git a/webservice/main.py b/webservice/main.py index b480265..443fb3a 100644 --- a/webservice/main.py +++ b/webservice/main.py @@ -62,7 +62,7 @@ class UserConfigServer(webapp2.RequestHandler): def getUserConfig(self,user,url,fileformat): - configuration = Configuration(adapters.getConfigSectionFor(url),fileformat) + configuration = Configuration(adapters.getConfigSectionsFor(url),fileformat) logging.debug('reading defaults.ini config file') configuration.read('fanficfare/defaults.ini') From ce2476d3a82dd4d3c3dd89c03282e1fb293cef46 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 29 Jun 2015 13:07:38 -0500 Subject: [PATCH 15/32] Add [base_efiction] and [base_xenforoforum] sections, add base_xenforoforum_adapter, document new options. --- .../adapters/base_xenforoforum_adapter.py | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 fanficfare/adapters/base_xenforoforum_adapter.py diff --git a/fanficfare/adapters/base_xenforoforum_adapter.py b/fanficfare/adapters/base_xenforoforum_adapter.py new file mode 100644 index 0000000..406ab25 --- /dev/null +++ b/fanficfare/adapters/base_xenforoforum_adapter.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 FanFicFare team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +logger = logging.getLogger(__name__) + +class BaseXenForoForumAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["utf8", + "Windows-1252"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2]) + + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + + # normalized story URL. + self._setURL(self.getURLPrefix() + '/'+m.group('tp')+'/'+self.story.getMetadata('storyId')+'/') + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','fsb') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%b %d, %Y at %I:%M %p" + + @classmethod + def getConfigSections(cls): + "Only needs to be overriden if has additional ini sections." + return ['base_xenforoforum',cls.getConfigSection()] + + @classmethod + def getURLPrefix(cls): + # The site domain. Does have www here, if it uses it. + return 'https://' + cls.getSiteDomain() + + @classmethod + def getSiteExampleURLs(cls): + return cls.getURLPrefix()+"/threads/some-story-name.123456/" + + def getSiteURLPattern(self): + return r"https?://"+re.escape(self.getSiteDomain())+r"/(?Pthreads|posts)/(.+\.)?(?P\d+)/" + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + useurl = self.url + logger.info("url: "+useurl) + + try: + (data,opened) = self._fetchUrlOpened(useurl) + useurl = opened.geturl() + logger.info("use useurl: "+useurl) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = self.make_soup(data) + + a = soup.find('h3',{'class':'userText'}).find('a') + self.story.addToList('authorId',a['href'].split('/')[1]) + self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href']) + self.story.addToList('author',a.text) + + h1 = soup.find('div',{'class':'titleBar'}).h1 + self.story.setMetadata('title',stripHTML(h1)) + + if '#' in useurl: + anchorid = useurl.split('#')[1] + soup = soup.find('li',id=anchorid) + else: + # try threadmarks if no '#' in , require at least 2. + threadmarksa = soup.find('a',{'class':'threadmarksTrigger'}) + if threadmarksa: + soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href'])) + markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a') + if len(markas) > 1: + for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]: + date = self.make_date(atag.find_next_sibling('div',{'class':'extra'})) + if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'): + self.story.setMetadata('datePublished', date) + if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'): + self.story.setMetadata('dateUpdated', date) + + self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) + + soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above) + + # Now go hunting for the 'chapter list'. + bq = soup.find('blockquote') # assume first posting contains TOC urls. + + bq.name='div' + + for iframe in bq.find_all('iframe'): + iframe.extract() # calibre book reader & editor don't like iframes to youtube. + + for qdiv in bq.find_all('div',{'class':'quoteExpand'}): + qdiv.extract() # Remove
click to expand
+ + self.setDescription(useurl,bq) + + # otherwise, use first post links--include first post since that's + if not self.chapterUrls: + self.chapterUrls.append(("First Post",useurl)) + for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]: + logger.debug("found chapurl:%s"%url) + if not url.startswith('http'): + url = self.getURLPrefix()+'/'+url + + if (url.startswith(self.getURLPrefix()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): + # brute force way to deal with SB's http->https change when hardcoded http urls. + url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix()) + logger.debug("used chapurl:%s"%(url)) + self.chapterUrls.append((name,url)) + if url == useurl and 'First Post' == self.chapterUrls[0][0]: + # remove "First Post" if included in list. + logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0]) + del self.chapterUrls[0] + + # Didn't use threadmarks, so take created/updated dates + # from the 'first' posting created and updated. + date = self.make_date(soup.find('a',{'class':'datePermalink'})) + if date: + self.story.setMetadata('datePublished', date) + self.story.setMetadata('dateUpdated', date) # updated overwritten below if found. + + date = self.make_date(soup.find('div',{'class':'editDate'})) + if date: + self.story.setMetadata('dateUpdated', date) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + def make_date(self,parenttag): # forums use a BS thing where dates + # can appear different if recent. + datestr=None + try: + datetag = parenttag.find('span',{'class':'DateTime'}) + if datetag: + datestr = datetag['title'] + else: + datetag = parenttag.find('abbr',{'class':'DateTime'}) + if datetag: + datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring']) + # Apr 24, 2015 at 4:39 AM + # May 1, 2015 at 5:47 AM + datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours. + return makeDate(datestr, self.dateformat) + except: + logger.debug('No date found in %s'%parenttag) + return None + + # grab the text for an individual chapter. + def getChapterText(self, url): + logger.debug('Getting chapter text from: %s' % url) + + (data,opened) = self._fetchUrlOpened(url) + url = opened.geturl() + logger.debug("chapter URL redirected to: %s"%url) + + soup = self.make_soup(data) + + if '#' in url: + anchorid = url.split('#')[1] + soup = soup.find('li',id=anchorid) + bq = soup.find('blockquote') + + bq.name='div' + + for iframe in bq.find_all('iframe'): + iframe.extract() # calibre book reader & editor don't like iframes to youtube. + + for qdiv in bq.find_all('div',{'class':'quoteExpand'}): + qdiv.extract() # Remove
click to expand
+ + return self.utf8FromSoup(url,bq) From a84c5dea191f9dc812084ebb79118f663d4967a5 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 30 Jun 2015 22:44:06 -0500 Subject: [PATCH 16/32] Remove outdated comment. --- fanficfare/writers/writer_epub.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fanficfare/writers/writer_epub.py b/fanficfare/writers/writer_epub.py index 52deae9..d59832b 100644 --- a/fanficfare/writers/writer_epub.py +++ b/fanficfare/writers/writer_epub.py @@ -661,10 +661,6 @@ div { margin: 0pt; padding: 0pt; } # as one line. This causes problems for nook(at # least) when the chapter size starts getting big # (200k+) - #fullhtml = fullhtml.replace('

','

\n').replace('
','
\n') - # The replaces above added tons of extra newlines - # during *each* epub update. The regexp version adds - # only one and removes any extra. fullhtml = re.sub(r'(

|
)\n*',r'\1\n',fullhtml) outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8')) From 21872601db4307232c651dfcae298720afdbe99d Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 3 Jul 2015 11:15:46 -0500 Subject: [PATCH 17/32] Allow https links from SV forum stories, too. --- fanficfare/adapters/base_xenforoforum_adapter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fanficfare/adapters/base_xenforoforum_adapter.py b/fanficfare/adapters/base_xenforoforum_adapter.py index 406ab25..8d830b9 100644 --- a/fanficfare/adapters/base_xenforoforum_adapter.py +++ b/fanficfare/adapters/base_xenforoforum_adapter.py @@ -155,7 +155,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): if not url.startswith('http'): url = self.getURLPrefix()+'/'+url - if (url.startswith(self.getURLPrefix()) or url.startswith('http://'+self.getSiteDomain())) and ('/posts/' in url or '/threads/' in url): + if ( url.startswith(self.getURLPrefix()) or + url.startswith('http://'+self.getSiteDomain()) or + url.startswith('https://'+self.getSiteDomain()) ) and ('/posts/' in url or '/threads/' in url): # brute force way to deal with SB's http->https change when hardcoded http urls. url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix()) logger.debug("used chapurl:%s"%(url)) From e637b4b073821d1c2ce776391938882b40fa8d66 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 3 Jul 2015 11:16:47 -0500 Subject: [PATCH 18/32] Save first version of UnNew feature for removing '(new)' marks on chapters. --- calibre-plugin/fff_plugin.py | 74 ++++++++++++++++++++- fanficfare/epubutils.py | 104 ++++++++++++++++++++++++++---- fanficfare/story.py | 21 +++--- fanficfare/writers/base_writer.py | 8 +-- fanficfare/writers/writer_epub.py | 26 +++++--- fanficfare/writers/writer_html.py | 10 +-- fanficfare/writers/writer_mobi.py | 10 +-- fanficfare/writers/writer_txt.py | 10 +-- 8 files changed, 214 insertions(+), 49 deletions(-) diff --git a/calibre-plugin/fff_plugin.py b/calibre-plugin/fff_plugin.py index 26503dc..cfab018 100644 --- a/calibre-plugin/fff_plugin.py +++ b/calibre-plugin/fff_plugin.py @@ -66,7 +66,8 @@ from calibre_plugins.fanficfare_plugin.fanficfare import ( adapters, exceptions) from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import ( - get_dcsource, get_dcsource_chaptercount, get_story_url_from_html) + get_dcsource, get_dcsource_chaptercount, get_story_url_from_html, + reset_orig_chapters_epub) from calibre_plugins.fanficfare_plugin.fanficfare.geturls import ( get_urls_from_page, get_urls_from_html,get_urls_from_text, @@ -335,6 +336,12 @@ class FanFicFarePlugin(InterfaceAction): image='minusminus.png', triggered=partial(self.update_lists,add=False)) + self.menu.addSeparator() + self.get_list_action = self.create_menu_item_ex(self.menu, _('UnNew Selected books'), + unique_name='Get URLs from Selected Books', + image='rotate-right.png', + triggered=self.unnew_books) + self.menu.addSeparator() self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'), unique_name='Get URLs from Selected Books', @@ -415,6 +422,7 @@ class FanFicFarePlugin(InterfaceAction): return self.update_reading_lists(self.gui.library_view.get_selected_ids(),add) + self.unnew_books() def get_urls_from_imap_menu(self): @@ -555,6 +563,70 @@ class FanFicFarePlugin(InterfaceAction): show=True, show_copy_button=False) + def unnew_books(self): + '''Get list of URLs from existing books.''' + if not self.is_library_view(): + self.gui.status_bar.show_message(_('Can only UnNew books in library'), + 3000) + return + + if not self.gui.current_view().selectionModel().selectedRows() : + self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'), + 3000) + return + + book_list = map( partial(self.make_book_id_only), + self.gui.library_view.get_selected_ids() ) + + tdir = PersistentTemporaryDirectory(prefix='fanficfare_') + LoopProgressDialog(self.gui, + book_list, + partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir), + partial(self.get_unnew_books_finish, tdir=tdir), + init_label=_("UnNewing books..."), + win_title=_("UnNew Books"), + status_prefix=_("Books UnNewed")) + + def get_unnew_books_loop(self,book,db=None,tdir=None): + + if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True): + tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'], + suffix='.epub', + dir=tdir) + db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True) + + unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'], + suffix='.epub', + dir=tdir) + book['changed']=reset_orig_chapters_epub(tmp,unnewtmp) + if book['changed']: + db.add_format_with_hooks(book['calibre_id'], + 'EPUB', + unnewtmp, + index_is_id=True) + if prefs['deleteotherforms']: + fmts = db.formats(book['calibre_id'], index_is_id=True).split(',') + for fmt in fmts: + if fmt.lower() != formmapping['epub'].lower(): + logger.debug("deleteotherforms remove f:"+fmt) + db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + elif prefs['autoconvert']: + ## 'Convert Book'.auto_convert_auto_add doesn't convert if + ## the format is already there. + fmt = calibre_prefs['output_format'] + # delete if there, but not if the format we just made. + if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True): + logger.debug("autoconvert remove f:"+fmt) + db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + + def get_unnew_books_finish(self, book_list, tdir=None): + remove_dir(tdir) + if prefs['autoconvert']: + changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ] + if changed_ids: + self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000) + self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids) + def reject_list_urls(self): if self.is_library_view(): book_list = map( partial(self.make_book_id_only), diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index 6e51164..8f92ca2 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) @@ -11,7 +12,8 @@ import logging logger = logging.getLogger(__name__) import re, os, traceback -from zipfile import ZipFile +from collections import defaultdict +from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED from xml.dom.minidom import parseString import bs4 as bs @@ -92,6 +94,7 @@ def get_update_data(inputio, soups = [] # list of xhmtl blocks urlsoups = {} # map of xhtml blocks by url images = {} # dict() longdesc->data + datamaps = defaultdict(dict) # map of data maps by url if getfilecount: # spin through the manifest--only place there are item tags. for item in contentdom.getElementsByTagName("item"): @@ -125,25 +128,45 @@ def get_update_data(inputio, logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc)) logger.warn("Exception: %s"%(unicode(e))) traceback.print_exc() - soup = soup.find('body') + bodysoup = soup.find('body') # ffdl epubs have chapter title h3 - h3 = soup.find('h3') + h3 = bodysoup.find('h3') if h3: h3.extract() # TtH epubs have chapter title h2 - h2 = soup.find('h2') + h2 = bodysoup.find('h2') if h2: h2.extract() - for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}): + for skip in bodysoup.findAll(attrs={'class':'skip_on_ffdl_update'}): skip.extract() - chapa = soup.find('a',{'class':'chapterurl'}) - if chapa and chapa['href'] not in urlsoups: # keep first found if more than one. - urlsoups[chapa['href']] = soup - chapa.extract() + ## + #print("look for meta chapurl") + currenturl = None + chapurl = soup.find('meta',{'name':'chapterurl'}) + if chapurl: + if chapurl['content'] not in urlsoups: # keep first found if more than one. + #print("Found chapurl['content']:%s"%chapurl['content']) + currenturl = chapurl['content'] + urlsoups[chapurl['content']] = bodysoup + else: + # for older pre-meta. Only temp. + chapa = bodysoup.find('a',{'class':'chapterurl'}) + if chapa and chapa['href'] not in urlsoups: # keep first found if more than one. + urlsoups[chapa['href']] = bodysoup + currenturl = chapa['href'] + chapa.extract() + + chapterorigtitle = soup.find('meta',{'name':'chapterorigtitle'}) + if chapterorigtitle: + datamaps[currenturl]['chapterorigtitle'] = chapterorigtitle['content'] - soups.append(soup) + chaptertitle = soup.find('meta',{'name':'chaptertitle'}) + if chaptertitle: + datamaps[currenturl]['chaptertitle'] = chaptertitle['content'] + + soups.append(bodysoup) filecount+=1 @@ -154,7 +177,8 @@ def get_update_data(inputio, #for k in images.keys(): #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k]))) - return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups) + print("datamaps:%s"%datamaps) + return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps) def get_path_part(n): relpath = os.path.dirname(n) @@ -198,3 +222,61 @@ def get_story_url_from_html(inputio,_is_good_url=None): if _is_good_url == None or _is_good_url(ahref): return ahref return None + +def reset_orig_chapters_epub(inputio,outputio): + inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob + + ## Write mimetype file, must be first and uncompressed. + ## Older versions of python(2.4/5) don't allow you to specify + ## compression by individual file. + ## Overwrite if existing output file. + outputepub = ZipFile(outputio, 'w', compression=ZIP_STORED) + outputepub.debug = 3 + outputepub.writestr("mimetype", "application/epub+zip") + outputepub.close() + + ## Re-open file for content. + outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED) + outputepub.debug = 3 + + changed = False + + tocncx = inputepub.read('toc.ncx').decode('utf-8') + ## spin through file contents. + for zf in inputepub.namelist(): + if zf not in ['mimetype','toc.ncx'] : + data = inputepub.read(zf) + if isinstance(data,unicode): + print("\n\n\ndata is unicode\n\n\n") + if re.match(r'.*/file\d+\.xhtml',zf): + data = data.decode('utf-8') + soup = bs.BeautifulSoup(data,"html5lib") + + chapterorigtitle = None + tag = soup.find('meta',{'name':'chapterorigtitle'}) + if tag: + chapterorigtitle = tag['content'] + + chaptertitle = None + tag = soup.find('meta',{'name':'chaptertitle'}) + if tag: + chaptertitle = tag['content'] + + if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle: + origdata = data + origtocncx = tocncx + print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle)) + # changed = True + # data = data.replace(u'', + # u''+chapterorigtitle+u'') + data = data.replace(u''+chaptertitle+u'',u''+chapterorigtitle+u'') + data = data.replace(u'

'+chaptertitle+u'

',u'

'+chapterorigtitle+u'

') + tocncx = tocncx.replace(u''+chaptertitle+u'',u''+chapterorigtitle+u'') + changed = ( origdata != data or origtocncx != tocncx ) + outputepub.writestr(zf,data.encode('utf-8')) + else: + outputepub.writestr(zf,data) + + outputepub.writestr('toc.ncx',tocncx.encode('utf-8')) + + return changed diff --git a/fanficfare/story.py b/fanficfare/story.py index dd578ce..017a469 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -16,6 +16,7 @@ # import os, re +from collections import namedtuple import urlparse import string import json @@ -32,6 +33,8 @@ import exceptions from htmlcleanup import conditionalRemoveEntities, removeAllEntities from configurable import Configurable, re_compile +Chapter = namedtuple('Chapter', 'url title html origtitle') + SPACE_REPLACE=u'\s' SPLIT_META=u'\,' @@ -412,7 +415,7 @@ class Story(Configurable): except: self.metadata = {'version':'4.4'} self.in_ex_cludes = {} - self.chapters = [] # chapters will be tuples of (url,title,html) + self.chapters = [] # chapters will be namedtuple of Chapter(url,title,html,etc) self.imgurls = [] self.imgtuples = [] @@ -844,22 +847,24 @@ class Story(Configurable): if self.getConfig('strip_chapter_numbers') and \ self.getConfig('chapter_title_strip_pattern'): title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title) + newtitle=title if marknewchap: - title=u'(new) %s'%title - self.chapters.append( (url,title,html) ) + newtitle=u'(new) %s'%title + self.chapters.append( Chapter(url,newtitle,html,title) ) def getChapters(self,fortoc=False): - "Chapters will be tuples of (title,html)" + "Chapters will be Chapter namedtuples of (url,title,html,new)" retval = [] ## only add numbers if more than one chapter. if len(self.chapters) > 1 and \ (self.getConfig('add_chapter_numbers') == "true" \ or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \ and self.getConfig('chapter_title_add_pattern'): - for index, (url,title,html) in enumerate(self.chapters): - retval.append( (url, - string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}), - html) ) + for index, chap in enumerate(self.chapters): + retval.append( Chapter(chap.url, + string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.title}), + chap.html, + string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.origtitle})) ) else: retval = self.chapters diff --git a/fanficfare/writers/base_writer.py b/fanficfare/writers/base_writer.py index 3fdde1c..df5885c 100644 --- a/fanficfare/writers/base_writer.py +++ b/fanficfare/writers/base_writer.py @@ -148,12 +148,12 @@ class BaseStoryWriter(Configurable): self._write(out,START.substitute(self.story.getAllMetadata())) - for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)): - if html: - self._write(out,ENTRY.substitute({'chapter':title, + for index, chap in enumerate(self.story.getChapters(fortoc=True)): + if chap.html: + self._write(out,ENTRY.substitute({'chapter':chap.title, 'number':index+1, 'index':"%04d"%(index+1), - 'url':url})) + 'url':chap.url})) self._write(out,END.substitute(self.story.getAllMetadata())) diff --git a/fanficfare/writers/writer_epub.py b/fanficfare/writers/writer_epub.py index d59832b..3e0fcfb 100644 --- a/fanficfare/writers/writer_epub.py +++ b/fanficfare/writers/writer_epub.py @@ -28,7 +28,7 @@ import re from xml.dom.minidom import parse, parseString, getDOMImplementation from base_writer import * -from ..htmlcleanup import stripHTML +from ..htmlcleanup import stripHTML,removeEntities logger = logging.getLogger(__name__) @@ -133,6 +133,9 @@ ${value}
${chapter} + + +

${chapter}

@@ -502,13 +505,13 @@ div { margin: 0pt; padding: 0pt; } items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log")) itemrefs.append("log_page") - for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)): - if html: + for index, chap in enumerate(self.story.getChapters(fortoc=True)): + if chap.html: i=index+1 items.append(("file%04d"%i, "OEBPS/file%04d.xhtml"%i, "application/xhtml+xml", - title)) + chap.title)) itemrefs.append("file%04d"%i) manifest = contentdom.createElement("manifest") @@ -650,13 +653,16 @@ div { margin: 0pt; padding: 0pt; } else: CHAPTER_END = self.EPUB_CHAPTER_END - for index, (url,title,html) in enumerate(self.story.getChapters()): - if html: - logger.debug('Writing chapter text for: %s' % title) - vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + for index, chap in enumerate(self.story.getChapters()): # (url,title,html) + if chap.html: + logger.debug('Writing chapter text for: %s' % chap.title) + vals={'url':removeEntities(chap.url), + 'chapter':chap.title, + 'origchapter':chap.origtitle, + 'index':"%04d"%(index+1), + 'number':index+1} fullhtml = CHAPTER_START.substitute(vals) + \ - '' + \ - html + CHAPTER_END.substitute(vals) + chap.html + CHAPTER_END.substitute(vals) # ffnet(& maybe others) gives the whole chapter text # as one line. This causes problems for nook(at # least) when the chapter size starts getting big diff --git a/fanficfare/writers/writer_html.py b/fanficfare/writers/writer_html.py index 03f151e..c99e396 100644 --- a/fanficfare/writers/writer_html.py +++ b/fanficfare/writers/writer_html.py @@ -128,12 +128,12 @@ ${output_css} else: CHAPTER_END = self.HTML_CHAPTER_END - for index, (url,title,html) in enumerate(self.story.getChapters()): - if html: - logging.debug('Writing chapter text for: %s' % title) - vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + for index, chap in enumerate(self.story.getChapters()): + if chap.html: + logging.debug('Writing chapter text for: %s' % chap.title) + vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1} self._write(out,CHAPTER_START.substitute(vals)) - self._write(out,html) + self._write(out,chap.html) self._write(out,CHAPTER_END.substitute(vals)) self._write(out,FILE_END.substitute(self.story.getAllMetadata())) diff --git a/fanficfare/writers/writer_mobi.py b/fanficfare/writers/writer_mobi.py index 1018b56..ca8ed5c 100644 --- a/fanficfare/writers/writer_mobi.py +++ b/fanficfare/writers/writer_mobi.py @@ -161,11 +161,11 @@ ${value}
else: CHAPTER_END = self.MOBI_CHAPTER_END - for index, (url,title,html) in enumerate(self.story.getChapters()): - if html: - logger.debug('Writing chapter text for: %s' % title) - vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} - fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals) + for index, chap in enumerate(self.story.getChapters()): + if chap.html: + logger.debug('Writing chapter text for: %s' % chap.title) + vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1} + fullhtml = CHAPTER_START.substitute(vals) + chap.html + CHAPTER_END.substitute(vals) # ffnet(& maybe others) gives the whole chapter text # as one line. This causes problems for nook(at # least) when the chapter size starts getting big diff --git a/fanficfare/writers/writer_txt.py b/fanficfare/writers/writer_txt.py index c91726d..91b3ac5 100644 --- a/fanficfare/writers/writer_txt.py +++ b/fanficfare/writers/writer_txt.py @@ -154,12 +154,12 @@ End file. else: CHAPTER_END = self.TEXT_CHAPTER_END - for index, (url, title,html) in enumerate(self.story.getChapters()): - if html: - logging.debug('Writing chapter text for: %s' % title) - vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + for index, chap in enumerate(self.story.getChapters()): + if chap.html: + logging.debug('Writing chapter text for: %s' % chap.title) + vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1} self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals))))) - self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width))) + self._write(out,self.lineends(html2text(chap.html,wrap_width=self.wrap_width))) self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals))))) self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata())))) From d065ae56d2cbdf1a5fb7c92df8ff8c5179a673d2 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 10 Jul 2015 10:40:17 -0500 Subject: [PATCH 19/32] Make UnNew on 'mark read' optional, add '--unnew' option to CLI. --- calibre-plugin/config.py | 6 +++++ calibre-plugin/fff_plugin.py | 9 ++++---- calibre-plugin/prefs.py | 1 + fanficfare/cli.py | 18 +++++++++++++-- fanficfare/epubutils.py | 45 +++++++++++++++++++++++++----------- 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index 0708124..3fc7c64 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -272,6 +272,7 @@ class ConfigWidget(QWidget): prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked() prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked() prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked() + prefs['autounnew'] = self.readinglist_tab.autounnew.isChecked() # personal.ini ini = self.personalini_tab.personalini @@ -781,6 +782,11 @@ class ReadingListTab(QWidget): self.addtolistsonread.setChecked(prefs['addtolistsonread']) self.l.addWidget(self.addtolistsonread) + self.autounnew = QCheckBox(_('Automatically run Remove "New" Chapter Marks when marking books "Read".'),self) + self.autounnew.setToolTip(_('Menu option to remove from "To Read" lists will also remove "(new)" chapter marks created by personal.ini mark_new_chapters setting.')) + self.autounnew.setChecked(prefs['autounnew']) + self.l.addWidget(self.autounnew) + self.l.insertStretch(-1) class CalibreCoverTab(QWidget): diff --git a/calibre-plugin/fff_plugin.py b/calibre-plugin/fff_plugin.py index feae7c2..d7241e1 100644 --- a/calibre-plugin/fff_plugin.py +++ b/calibre-plugin/fff_plugin.py @@ -337,9 +337,9 @@ class FanFicFarePlugin(InterfaceAction): triggered=partial(self.update_lists,add=False)) self.menu.addSeparator() - self.get_list_action = self.create_menu_item_ex(self.menu, _('UnNew Selected books'), - unique_name='Get URLs from Selected Books', - image='rotate-right.png', + self.get_list_action = self.create_menu_item_ex(self.menu, _('Remove "New" Chapter Marks from Selected books'), + unique_name='Remove "(new)" chapter marks created by personal.ini mark_new_chapters setting.', + image='edit-undo.png', triggered=self.unnew_books) self.menu.addSeparator() @@ -422,7 +422,8 @@ class FanFicFarePlugin(InterfaceAction): return self.update_reading_lists(self.gui.library_view.get_selected_ids(),add) - self.unnew_books() + if not add and prefs['autounnew']: + self.unnew_books() def get_urls_from_imap_menu(self): diff --git a/calibre-plugin/prefs.py b/calibre-plugin/prefs.py index 69496c8..0dd8afd 100644 --- a/calibre-plugin/prefs.py +++ b/calibre-plugin/prefs.py @@ -85,6 +85,7 @@ default_prefs['read_lists'] = '' default_prefs['addtolists'] = False default_prefs['addtoreadlists'] = False default_prefs['addtolistsonread'] = False +default_prefs['autounnew'] = False default_prefs['updatecalcover'] = None default_prefs['gencalcover'] = SAVE_YES diff --git a/fanficfare/cli.py b/fanficfare/cli.py index 32ae102..f9ab6d3 100644 --- a/fanficfare/cli.py +++ b/fanficfare/cli.py @@ -41,12 +41,14 @@ try: # running under calibre from calibre_plugins.fanfictiondownloader_plugin.fanficfare import adapters, writers, exceptions from calibre_plugins.fanfictiondownloader_plugin.fanficfare.configurable import Configuration - from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import get_dcsource_chaptercount, get_update_data + from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import ( + get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub) from calibre_plugins.fanfictiondownloader_plugin.fanficfare.geturls import get_urls_from_page except ImportError: from fanficfare import adapters, writers, exceptions from fanficfare.configurable import Configuration - from fanficfare.epubutils import get_dcsource_chaptercount, get_update_data + from fanficfare.epubutils import ( + get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub) from fanficfare.geturls import get_urls_from_page @@ -87,6 +89,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non parser.add_option('-u', '--update-epub', action='store_true', dest='update', help='Update an existing epub with new chapters, give epub filename instead of storyurl.', ) + parser.add_option('--unnew', + action='store_true', dest='unnew', + help='Remove (new) chapter marks left by mark_new_chapters setting.', ) parser.add_option('--update-cover', action='store_true', dest='updatecover', help='Update cover in an existing epub, otherwise existing cover (if any) is used on update. Only valid with --update-epub.', ) @@ -129,6 +134,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non if options.update and options.format != 'epub': parser.error('-u/--update-epub only works with epub') + if options.unnew and options.format != 'epub': + parser.error('--unnew-epub only works with epub') + # for passing in a file list if options.infile: urls=[] @@ -168,6 +176,12 @@ def do_download(arg, # Attempt to update an existing epub. chaptercount = None output_filename = None + + if options.unnew: + # remove mark_new_chapters marks + reset_orig_chapters_epub(arg,arg) + return + if options.update: try: url, chaptercount = get_dcsource_chaptercount(arg) diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index 8f92ca2..deaa908 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -1,11 +1,10 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai # -*- coding: utf-8 -*- -from __future__ import (unicode_literals, division, absolute_import, - print_function) + +# from __future__ import (unicode_literals, division, absolute_import, +# print_function) __license__ = 'GPL v3' -__copyright__ = '2014, Jim Miller' +__copyright__ = '2015, Jim Miller' __docformat__ = 'restructuredtext en' import logging @@ -15,6 +14,7 @@ import re, os, traceback from collections import defaultdict from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED from xml.dom.minidom import parseString +from StringIO import StringIO import bs4 as bs @@ -177,7 +177,7 @@ def get_update_data(inputio, #for k in images.keys(): #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k]))) - print("datamaps:%s"%datamaps) + # print("datamaps:%s"%datamaps) return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps) def get_path_part(n): @@ -223,20 +223,23 @@ def get_story_url_from_html(inputio,_is_good_url=None): return ahref return None -def reset_orig_chapters_epub(inputio,outputio): +def reset_orig_chapters_epub(inputio,outfile): inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob + ## build zip in memory in case updating in place(CLI). + zipio = StringIO() + ## Write mimetype file, must be first and uncompressed. ## Older versions of python(2.4/5) don't allow you to specify ## compression by individual file. ## Overwrite if existing output file. - outputepub = ZipFile(outputio, 'w', compression=ZIP_STORED) + outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED) outputepub.debug = 3 outputepub.writestr("mimetype", "application/epub+zip") outputepub.close() ## Re-open file for content. - outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED) + outputepub = ZipFile(zipio, "a", compression=ZIP_DEFLATED) outputepub.debug = 3 changed = False @@ -246,8 +249,8 @@ def reset_orig_chapters_epub(inputio,outputio): for zf in inputepub.namelist(): if zf not in ['mimetype','toc.ncx'] : data = inputepub.read(zf) - if isinstance(data,unicode): - print("\n\n\ndata is unicode\n\n\n") + # if isinstance(data,unicode): + # logger.debug("\n\n\ndata is unicode\n\n\n") if re.match(r'.*/file\d+\.xhtml',zf): data = data.decode('utf-8') soup = bs.BeautifulSoup(data,"html5lib") @@ -265,7 +268,7 @@ def reset_orig_chapters_epub(inputio,outputio): if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle: origdata = data origtocncx = tocncx - print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle)) + # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle)) # changed = True # data = data.replace(u'', # u''+chapterorigtitle+u'') @@ -277,6 +280,22 @@ def reset_orig_chapters_epub(inputio,outputio): else: outputepub.writestr(zf,data) - outputepub.writestr('toc.ncx',tocncx.encode('utf-8')) + # only write if changed. + if changed: + outputepub.writestr('toc.ncx',tocncx.encode('utf-8')) + + # declares all the files created by Windows. otherwise, when + # it runs in appengine, windows unzips the files as 000 perms. + for zf in outputepub.filelist: + zf.create_system = 0 + outputepub.close() + if isinstance(outfile,basestring): + with open(outfile,"wb") as outputio: + outputio.write(zipio.getvalue()) + else: + outfile.write(zipio.getvalue()) + + inputepub.close() + zipio.close() return changed From 26e54b3fcb537c93bcbd28fcd040f87f6cda67e2 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 10 Jul 2015 10:55:46 -0500 Subject: [PATCH 20/32] Remove unneeded vim & #! comments. --- calibre-plugin/basicinihighlighter.py | 4 ++-- calibre-plugin/common_utils.py | 4 ++-- calibre-plugin/config.py | 4 ++-- calibre-plugin/dialogs.py | 4 ++-- calibre-plugin/fff_plugin.py | 4 ++-- calibre-plugin/fff_util.py | 4 ++-- calibre-plugin/inihighlighter.py | 4 ++-- calibre-plugin/jobs.py | 4 ++-- calibre-plugin/prefs.py | 4 ++-- fanficfare/epubutils.py | 3 --- 10 files changed, 18 insertions(+), 21 deletions(-) diff --git a/calibre-plugin/basicinihighlighter.py b/calibre-plugin/basicinihighlighter.py index c4ea2c9..b948a56 100644 --- a/calibre-plugin/basicinihighlighter.py +++ b/calibre-plugin/basicinihighlighter.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, print_function) diff --git a/calibre-plugin/common_utils.py b/calibre-plugin/common_utils.py index 9f135b4..d032027 100644 --- a/calibre-plugin/common_utils.py +++ b/calibre-plugin/common_utils.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, absolute_import, print_function) diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index 3fc7c64..f04c9fd 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, absolute_import, print_function) diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index 7735774..3b35821 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, print_function) diff --git a/calibre-plugin/fff_plugin.py b/calibre-plugin/fff_plugin.py index d7241e1..37ddde3 100644 --- a/calibre-plugin/fff_plugin.py +++ b/calibre-plugin/fff_plugin.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, absolute_import, print_function) diff --git a/calibre-plugin/fff_util.py b/calibre-plugin/fff_util.py index 0ba7227..be35131 100644 --- a/calibre-plugin/fff_util.py +++ b/calibre-plugin/fff_util.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, absolute_import, print_function) diff --git a/calibre-plugin/inihighlighter.py b/calibre-plugin/inihighlighter.py index 2f34198..7348d80 100644 --- a/calibre-plugin/inihighlighter.py +++ b/calibre-plugin/inihighlighter.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, print_function) diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index f8ea695..b962d9e 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, absolute_import, print_function) diff --git a/calibre-plugin/prefs.py b/calibre-plugin/prefs.py index 0dd8afd..cc5350f 100644 --- a/calibre-plugin/prefs.py +++ b/calibre-plugin/prefs.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- + from __future__ import (unicode_literals, division, absolute_import, print_function) diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index deaa908..5f02111 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -1,8 +1,5 @@ # -*- coding: utf-8 -*- -# from __future__ import (unicode_literals, division, absolute_import, -# print_function) - __license__ = 'GPL v3' __copyright__ = '2015, Jim Miller' __docformat__ = 'restructuredtext en' From 58f093072a5b2694284c2c51e65706b672e767cb Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 10 Jul 2015 12:40:54 -0500 Subject: [PATCH 21/32] Keep existing (new) chapter marks when updating epub. --- calibre-plugin/jobs.py | 3 ++- fanficfare/adapters/base_adapter.py | 14 +++++++++++--- fanficfare/cli.py | 3 ++- fanficfare/epubutils.py | 4 ++-- fanficfare/story.py | 15 ++++++++------- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index b962d9e..de8e017 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -193,7 +193,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): adapter.oldcover, adapter.calibrebookmark, adapter.logfile, - adapter.oldchaptersmap) = get_update_data(book['epub_for_update'])[0:8] + adapter.oldchaptersmap, + adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9] # dup handling from fff_plugin needed for anthology updates. if options['collision'] == UPDATE: diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index aebf330..9375aca 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -105,6 +105,7 @@ class BaseSiteAdapter(Configurable): self.chapterLast = None self.oldchapters = None self.oldchaptersmap = None + self.oldchaptersdata = None self.oldimgs = None self.oldcover = None # (data of existing cover html, data of existing cover image) self.calibrebookmark = None @@ -361,7 +362,7 @@ class BaseSiteAdapter(Configurable): self.getStoryMetadataOnly(get_cover=True) for index, (title,url) in enumerate(self.chapterUrls): - marknewchap = False + newchap = False if (self.chapterFirst!=None and index < self.chapterFirst) or \ (self.chapterLast!=None and index > self.chapterLast): self.story.addChapter(url, @@ -378,14 +379,21 @@ class BaseSiteAdapter(Configurable): data = self.utf8FromSoup(None, self.oldchapters[index], partial(cachedfetch,self._fetchUrlRaw,self.oldimgs)) + + newchap = (self.oldchaptersdata and + url in self.oldchaptersdata and ( + self.oldchaptersdata[url]['chapterorigtitle'] != + self.oldchaptersdata[url]['chaptertitle']) ) + if not data: data = self.getChapterText(url) # if configured and has existing chapters - marknewchap = (self.getConfig('mark_new_chapters')=='true' and self.oldchapters or self.oldchaptersmap) + newchap = (self.oldchapters or self.oldchaptersmap) + self.story.addChapter(url, removeEntities(title), removeEntities(data), - marknewchap) + newchap) self.storyDone = True # include image, but no cover from story, add default_cover_image cover. diff --git a/fanficfare/cli.py b/fanficfare/cli.py index f9ab6d3..d1ecefd 100644 --- a/fanficfare/cli.py +++ b/fanficfare/cli.py @@ -339,7 +339,8 @@ def do_download(arg, adapter.oldcover, adapter.calibrebookmark, adapter.logfile, - adapter.oldchaptersmap) = (get_update_data(output_filename))[0:8] + adapter.oldchaptersmap, + adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9] print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount) diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index 5f02111..45fb65d 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -267,8 +267,8 @@ def reset_orig_chapters_epub(inputio,outfile): origtocncx = tocncx # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle)) # changed = True - # data = data.replace(u'', - # u''+chapterorigtitle+u'') + data = data.replace(u'', + u'') data = data.replace(u''+chaptertitle+u'',u''+chapterorigtitle+u'') data = data.replace(u'

'+chaptertitle+u'

',u'

'+chapterorigtitle+u'

') tocncx = tocncx.replace(u''+chaptertitle+u'',u''+chapterorigtitle+u'') diff --git a/fanficfare/story.py b/fanficfare/story.py index 017a469..a39fd94 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -33,7 +33,7 @@ import exceptions from htmlcleanup import conditionalRemoveEntities, removeAllEntities from configurable import Configurable, re_compile -Chapter = namedtuple('Chapter', 'url title html origtitle') +Chapter = namedtuple('Chapter', 'url title html origtitle new') SPACE_REPLACE=u'\s' SPLIT_META=u'\,' @@ -843,19 +843,19 @@ class Story(Configurable): return list(subjectset | set(self.getConfigList("extratags"))) - def addChapter(self, url, title, html, marknewchap=False): + def addChapter(self, url, title, html, newchap=False): if self.getConfig('strip_chapter_numbers') and \ self.getConfig('chapter_title_strip_pattern'): title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title) newtitle=title - if marknewchap: + if newchap and self.getConfig('mark_new_chapters')=='true': newtitle=u'(new) %s'%title - self.chapters.append( Chapter(url,newtitle,html,title) ) + self.chapters.append( Chapter(url,newtitle,html,title,newchap) ) def getChapters(self,fortoc=False): - "Chapters will be Chapter namedtuples of (url,title,html,new)" + "Chapters will be Chapter namedtuples" retval = [] - ## only add numbers if more than one chapter. + ## only add numbers if more than one chapter. Ditto (new) marks. if len(self.chapters) > 1 and \ (self.getConfig('add_chapter_numbers') == "true" \ or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \ @@ -864,7 +864,8 @@ class Story(Configurable): retval.append( Chapter(chap.url, string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.title}), chap.html, - string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.origtitle})) ) + string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.origtitle}), + chap.new) ) else: retval = self.chapters From fbf409ccc50b138c29f16e5ef9a3ebb6c1d243c7 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 10 Jul 2015 16:38:08 -0500 Subject: [PATCH 22/32] Add ini chapter title patterns for default, add, new and addnew. --- calibre-plugin/plugin-defaults.ini | 32 +++++++++++------ fanficfare/adapters/base_adapter.py | 14 ++++---- fanficfare/defaults.ini | 32 +++++++++++------ fanficfare/epubutils.py | 51 +++++++++++++++++++------- fanficfare/story.py | 56 +++++++++++++++++++++++------ fanficfare/writers/writer_epub.py | 2 ++ 6 files changed, 137 insertions(+), 50 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index 329eac5..449a33d 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -293,18 +293,30 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+ ## etc #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)? -## Uses a python template substitution. The ${index} is the 'chapter' -## number and ${title} is the chapter title, after applying -## chapter_title_strip_pattern. Those are the only variables available. -## "The Beginning" => "1. The Beginning" +## If true, when updating an epub that already has old chapters, new +## chapters will be marked in the TOC and chapter header by using +## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter. +mark_new_chapters:false + +## chapter title patterns use python template substitution. The +## ${index} is the 'chapter' number and ${title} is the chapter title, +## after applying chapter_title_strip_pattern. Those are the only +## variables available. + +## The basic pattern used when not using add_chapter_numbers or +## mark_new_chapters +chapter_title_def_pattern:${title} + +## Pattern used with add_chapter_numbers, but not mark_new_chapters chapter_title_add_pattern:${index}. ${title} -## If true, when updating an epub that already has old chapters, new -## chapters will be marked in the TOC and chapter header by prepending -## '(new) ' to the chapter title. So 'The Big Fight' will become -## '4. (new) The Big Fight' if both mark_new_chapters and -## add_chapter_numbers are set true. -mark_new_chapters:false +## Pattern used with mark_new_chapters, but not add_chapter_numbers +## (new) is just text and can be changed. +chapter_title_new_pattern:(new) ${title} + +## Pattern used with add_chapter_numbers and mark_new_chapters +## (new) is just text and can be changed. +chapter_title_addnew_pattern:${index}. (new) ${title} ## Uses a python template substitution. The ${title} is the default ## title of a new anthology, in the case of a series, or diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 9375aca..3bb7669 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -379,16 +379,18 @@ class BaseSiteAdapter(Configurable): data = self.utf8FromSoup(None, self.oldchapters[index], partial(cachedfetch,self._fetchUrlRaw,self.oldimgs)) - - newchap = (self.oldchaptersdata and + + # if already marked new -- ie, origtitle and title don't match + # logger.debug("self.oldchaptersdata[url]:%s"%(self.oldchaptersdata[url])) + newchap = (self.oldchaptersdata is not None and url in self.oldchaptersdata and ( - self.oldchaptersdata[url]['chapterorigtitle'] != - self.oldchaptersdata[url]['chaptertitle']) ) + self.oldchaptersdata[url]['chapterorigtitle'] != + self.oldchaptersdata[url]['chaptertitle']) ) if not data: data = self.getChapterText(url) - # if configured and has existing chapters - newchap = (self.oldchapters or self.oldchaptersmap) + # if had to fetch and has existing chapters + newchap = bool(self.oldchapters or self.oldchaptersmap) self.story.addChapter(url, removeEntities(title), diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 26de893..e1ceed6 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -300,18 +300,30 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+ ## etc #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)? -## Uses a python template substitution. The ${index} is the 'chapter' -## number and ${title} is the chapter title, after applying -## chapter_title_strip_pattern. Those are the only variables available. -## "The Beginning" => "1. The Beginning" +## If true, when updating an epub that already has old chapters, new +## chapters will be marked in the TOC and chapter header by using +## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter. +mark_new_chapters:false + +## chapter title patterns use python template substitution. The +## ${index} is the 'chapter' number and ${title} is the chapter title, +## after applying chapter_title_strip_pattern. Those are the only +## variables available. + +## The basic pattern used when not using add_chapter_numbers or +## mark_new_chapters +chapter_title_def_pattern:${title} + +## Pattern used with add_chapter_numbers, but not mark_new_chapters chapter_title_add_pattern:${index}. ${title} -## If true, when updating an epub that already has old chapters, new -## chapters will be marked in the TOC and chapter header by prepending -## '(new) ' to the chapter title. So 'The Big Fight' will become -## '4. (new) The Big Fight' if both mark_new_chapters and -## add_chapter_numbers are set true. -mark_new_chapters:false +## Pattern used with mark_new_chapters, but not add_chapter_numbers +## (new) is just text and can be changed. +chapter_title_new_pattern:(new) ${title} + +## Pattern used with add_chapter_numbers and mark_new_chapters +## (new) is just text and can be changed. +chapter_title_addnew_pattern:${index}. (new) ${title} ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates ## on '/', so use replace_metadata to change separator first if diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index 45fb65d..963e66c 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -241,10 +241,11 @@ def reset_orig_chapters_epub(inputio,outfile): changed = False - tocncx = inputepub.read('toc.ncx').decode('utf-8') + tocncxdom = parseString(inputepub.read('toc.ncx')) ## spin through file contents. for zf in inputepub.namelist(): if zf not in ['mimetype','toc.ncx'] : + entrychanged = False data = inputepub.read(zf) # if isinstance(data,unicode): # logger.debug("\n\n\ndata is unicode\n\n\n") @@ -256,6 +257,14 @@ def reset_orig_chapters_epub(inputio,outfile): tag = soup.find('meta',{'name':'chapterorigtitle'}) if tag: chapterorigtitle = tag['content'] + + # toctitle is separate for add_chapter_numbers:toconly users. + chaptertoctitle = None + tag = soup.find('meta',{'name':'chaptertoctitle'}) + if tag: + chaptertoctitle = tag['content'] + elif chapterorigtitle: + chaptertoctitle = chapterorigtitle chaptertitle = None tag = soup.find('meta',{'name':'chaptertitle'}) @@ -264,28 +273,44 @@ def reset_orig_chapters_epub(inputio,outfile): if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle: origdata = data - origtocncx = tocncx # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle)) - # changed = True data = data.replace(u'', u'') data = data.replace(u''+chaptertitle+u'',u''+chapterorigtitle+u'') data = data.replace(u'

'+chaptertitle+u'

',u'

'+chapterorigtitle+u'

') - tocncx = tocncx.replace(u''+chaptertitle+u'',u''+chapterorigtitle+u'') - changed = ( origdata != data or origtocncx != tocncx ) + + entrychanged = ( origdata != data ) + changed = changed or entrychanged + + if entrychanged: + ## go after the TOC entry, too. + # + # + # 5. (new) Chapter 4 + # + # + # + for contenttag in tocncxdom.getElementsByTagName("content"): + if contenttag.getAttribute('src') == zf: + texttag = contenttag.parentNode.getElementsByTagName('navLabel')[0].getElementsByTagName('text')[0] + texttag.childNodes[0].replaceWholeText(chaptertoctitle) + # logger.debug("text label:%s"%texttag.toxml()) + continue + outputepub.writestr(zf,data.encode('utf-8')) else: + # possibly binary data, thus no .encode(). outputepub.writestr(zf,data) - # only write if changed. + outputepub.writestr('toc.ncx',tocncxdom.toxml(encoding='utf-8')) + outputepub.close() + # declares all the files created by Windows. otherwise, when + # it runs in appengine, windows unzips the files as 000 perms. + for zf in outputepub.filelist: + zf.create_system = 0 + + # only *actually* write if changed. if changed: - outputepub.writestr('toc.ncx',tocncx.encode('utf-8')) - - # declares all the files created by Windows. otherwise, when - # it runs in appengine, windows unzips the files as 000 perms. - for zf in outputepub.filelist: - zf.create_system = 0 - outputepub.close() if isinstance(outfile,basestring): with open(outfile,"wb") as outputio: outputio.write(zipio.getvalue()) diff --git a/fanficfare/story.py b/fanficfare/story.py index a39fd94..9c7edc3 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -33,7 +33,7 @@ import exceptions from htmlcleanup import conditionalRemoveEntities, removeAllEntities from configurable import Configurable, re_compile -Chapter = namedtuple('Chapter', 'url title html origtitle new') +Chapter = namedtuple('Chapter', 'url title html origtitle toctitle new') SPACE_REPLACE=u'\s' SPLIT_META=u'\,' @@ -844,27 +844,61 @@ class Story(Configurable): return list(subjectset | set(self.getConfigList("extratags"))) def addChapter(self, url, title, html, newchap=False): + # logger.debug("addChapter(%s,%s)"%(url,newchap)) if self.getConfig('strip_chapter_numbers') and \ self.getConfig('chapter_title_strip_pattern'): title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title) - newtitle=title - if newchap and self.getConfig('mark_new_chapters')=='true': - newtitle=u'(new) %s'%title - self.chapters.append( Chapter(url,newtitle,html,title,newchap) ) + self.chapters.append( Chapter(url,title,html,title,title,newchap) ) def getChapters(self,fortoc=False): "Chapters will be Chapter namedtuples" retval = [] + ## only add numbers if more than one chapter. Ditto (new) marks. - if len(self.chapters) > 1 and \ - (self.getConfig('add_chapter_numbers') == "true" \ - or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \ - and self.getConfig('chapter_title_add_pattern'): + if len(self.chapters) > 1: + addnums = ( self.getConfig('add_chapter_numbers') == "true" + or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc) ) + + marknew = self.getConfig('mark_new_chapters')=='true' + + defpattern = self.getConfig('chapter_title_def_pattern','${title}') # default val in case of missing defaults.ini + if addnums and marknew: + pattern = self.getConfig('chapter_title_add_pattern') + newpattern = self.getConfig('chapter_title_addnew_pattern') + elif addnums: + pattern = self.getConfig('chapter_title_add_pattern') + newpattern = pattern + elif marknew: + pattern = defpattern + newpattern = self.getConfig('chapter_title_new_pattern') + else: + pattern = defpattern + newpattern = pattern + + if self.getConfig('add_chapter_numbers') in ["true","toconly"]: + tocpattern = self.getConfig('chapter_title_add_pattern') + else: + tocpattern = defpattern + + # logger.debug("Patterns: (%s)(%s)"%(pattern,newpattern)) + templ = string.Template(pattern) + newtempl = string.Template(newpattern) + toctempl = string.Template(tocpattern) + for index, chap in enumerate(self.chapters): + if chap.new: + usetempl = newtempl + else: + usetempl = templ + # logger.debug("chap.url, chap.new: (%s)(%s)"%(chap.url,chap.new)) retval.append( Chapter(chap.url, - string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.title}), + # 'new' + usetempl.substitute({'index':index+1,'title':chap.title}), chap.html, - string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.origtitle}), + # 'orig' + templ.substitute({'index':index+1,'title':chap.title}), + # 'toc' + toctempl.substitute({'index':index+1,'title':chap.title}), chap.new) ) else: retval = self.chapters diff --git a/fanficfare/writers/writer_epub.py b/fanficfare/writers/writer_epub.py index 3e0fcfb..4364571 100644 --- a/fanficfare/writers/writer_epub.py +++ b/fanficfare/writers/writer_epub.py @@ -135,6 +135,7 @@ ${value}
+ @@ -659,6 +660,7 @@ div { margin: 0pt; padding: 0pt; } vals={'url':removeEntities(chap.url), 'chapter':chap.title, 'origchapter':chap.origtitle, + 'tocchapter':chap.toctitle, 'index':"%04d"%(index+1), 'number':index+1} fullhtml = CHAPTER_START.substitute(vals) + \ From b2cc7053eead6b524ad4931ef8449e90ceae489d Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 10 Jul 2015 17:58:01 -0500 Subject: [PATCH 23/32] Minor tweak fixes. --- calibre-plugin/plugin-defaults.ini | 8 +++++--- fanficfare/cli.py | 2 +- fanficfare/configurable.py | 5 ++++- fanficfare/defaults.ini | 8 +++++--- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index 449a33d..ca9edb2 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -383,11 +383,11 @@ tagsfromtitle_label:Tags from Title ## also often include non-category stuff. # include_in_category:tagsfromtitle -include_metadata_pre: +add_to_include_metadata_pre: # only keep tagsfromtitle with ( or [ in. tagsfromtitle=~[\[\(] -replace_metadata: +add_to_replace_metadata: # remove anything outside () or [] tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 # remove () [] @@ -402,7 +402,7 @@ replace_metadata: # four", "Thread iv", etc title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> -extra_titlepage_entries: tagsfromtitle +add_to_extra_titlepage_entries:,tagsfromtitle ## '.SPLIT' tells the system to split by ',' add_to_include_subject_tags:,tagsfromtitle.SPLIT @@ -665,6 +665,8 @@ extratags: FanFiction,Testing,HTML ## doesn't like that. If do_update_hook is uncommented and set true, ## the adapter will discard all existing chapters from the newest one ## on when updating to enforce accurate chapters. +## Starting July 2015, FFF stores chapter URLs in the chapter files. +## Stories downloaded after that shouldn't need this setting anymore. #do_update_hook:false ## AO3 adapter defines a few extra metadata entries. diff --git a/fanficfare/cli.py b/fanficfare/cli.py index d1ecefd..2291d12 100644 --- a/fanficfare/cli.py +++ b/fanficfare/cli.py @@ -135,7 +135,7 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non parser.error('-u/--update-epub only works with epub') if options.unnew and options.format != 'epub': - parser.error('--unnew-epub only works with epub') + parser.error('--unnew only works with epub') # for passing in a file list if options.infile: diff --git a/fanficfare/configurable.py b/fanficfare/configurable.py index f38f474..97fc399 100644 --- a/fanficfare/configurable.py +++ b/fanficfare/configurable.py @@ -213,8 +213,11 @@ def get_valid_keywords(): 'bulk_load', 'chapter_end', 'chapter_start', - 'chapter_title_add_pattern', 'chapter_title_strip_pattern', + 'chapter_title_def_pattern', + 'chapter_title_add_pattern', + 'chapter_title_new_pattern', + 'chapter_title_addnew_pattern', 'mark_new_chapters', 'check_next_chapter', 'skip_author_cover', diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index e1ceed6..ab2a662 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -380,11 +380,11 @@ tagsfromtitle_label:Tags from Title ## also often include non-category stuff. # include_in_category:tagsfromtitle -include_metadata_pre: +add_to_include_metadata_pre: # only keep tagsfromtitle with ( or [ in. tagsfromtitle=~[\[\(] -replace_metadata: +add_to_replace_metadata: # remove anything outside () or [] tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1 # remove () [] @@ -399,7 +399,7 @@ replace_metadata: # four", "Thread iv", etc title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> -extra_titlepage_entries: tagsfromtitle +add_to_extra_titlepage_entries:,tagsfromtitle ## '.SPLIT' tells the system to split by ',' add_to_include_subject_tags:,tagsfromtitle.SPLIT @@ -669,6 +669,8 @@ extratags: FanFiction,Testing,HTML ## doesn't like that. If do_update_hook is uncommented and set true, ## the adapter will discard all existing chapters from the newest one ## on when updating to enforce accurate chapters. +## Starting July 2015, FFF stores chapter URLs in the chapter files. +## Stories downloaded after that shouldn't need this setting anymore. #do_update_hook:false ## AO3 adapter defines a few extra metadata entries. From 19109bfb4e6dbf95359a245fcdc34f021113ca16 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Sun, 12 Jul 2015 15:41:43 -0500 Subject: [PATCH 24/32] Improved comment. --- fanficfare/adapters/base_xenforoforum_adapter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fanficfare/adapters/base_xenforoforum_adapter.py b/fanficfare/adapters/base_xenforoforum_adapter.py index 8d830b9..4429080 100644 --- a/fanficfare/adapters/base_xenforoforum_adapter.py +++ b/fanficfare/adapters/base_xenforoforum_adapter.py @@ -147,7 +147,8 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): self.setDescription(useurl,bq) - # otherwise, use first post links--include first post since that's + # otherwise, use first post links--include first post since + # that's often also the first chapter. if not self.chapterUrls: self.chapterUrls.append(("First Post",useurl)) for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]: From d633f70ce7e5ab1c35e03c102728042d3db4c843 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 14 Jul 2015 13:27:22 -0500 Subject: [PATCH 25/32] Tweak forums title replace_metadata --- calibre-plugin/plugin-defaults.ini | 4 ++-- fanficfare/defaults.ini | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index ca9edb2..7252f85 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -396,8 +396,8 @@ add_to_replace_metadata: tagsfromtitle=> */ *=>, tagsfromtitle=> x =>, -# remove [] or () blocks and leading/trailing spaces - title=> *[\(\[]([^\]\)]+)[\)\]] *=> +# remove [] or () blocks and leading/trailing spaces/dashes/colons + title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=> # remove 'Thread' and the next word, usually "Thread 2", "Thread # four", "Thread iv", etc title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index ab2a662..9f66a25 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -393,8 +393,8 @@ add_to_replace_metadata: tagsfromtitle=> */ *=>, tagsfromtitle=> x =>, -# remove [] or () blocks and leading/trailing spaces - title=> *[\(\[]([^\]\)]+)[\)\]] *=> +# remove [] or () blocks and leading/trailing spaces/dashes/colons + title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=> # remove 'Thread' and the next word, usually "Thread 2", "Thread # four", "Thread iv", etc title=>[-: ]*[Tt]hread [^ ]+[-: ]*=> From 48749cfc2e64c0b7fdce97df657c61360aad3d6f Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 14 Jul 2015 13:46:11 -0500 Subject: [PATCH 26/32] Update messages.pot for translations. --- calibre-plugin/translations/messages.pot | 1178 +++++++++++----------- 1 file changed, 603 insertions(+), 575 deletions(-) diff --git a/calibre-plugin/translations/messages.pot b/calibre-plugin/translations/messages.pot index 0666ee0..eee5154 100644 --- a/calibre-plugin/translations/messages.pot +++ b/calibre-plugin/translations/messages.pot @@ -5,7 +5,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" -"POT-Creation-Date: 2015-07-07 23:14+Central Daylight Time\n" +"POT-Creation-Date: 2015-07-14 13:44+Central Daylight Time\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -103,65 +103,65 @@ msgstr "" msgid "Other" msgstr "" -#: config.py:380 +#: config.py:381 msgid "These settings control the basic features of the plugin--downloading FanFiction." msgstr "" -#: config.py:384 +#: config.py:385 msgid "Defaults Options on Download" msgstr "" -#: config.py:388 +#: config.py:389 msgid "On each download, FanFicFare offers an option to select the output format.
This sets what that option will default to." msgstr "" -#: config.py:390 +#: config.py:391 msgid "Default Output &Format:" msgstr "" -#: config.py:405 +#: config.py:406 msgid "On each download, FanFicFare offers an option of what happens if that story already exists.
This sets what that option will default to." msgstr "" -#: config.py:407 +#: config.py:408 msgid "Default If Story Already Exists?" msgstr "" -#: config.py:422 +#: config.py:423 msgid "Default Update Calibre &Metadata?" msgstr "" -#: config.py:423 +#: config.py:424 msgid "On each download, FanFicFare offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site.
This sets whether that will default to on or off.
Columns set to 'New Only' in the column tabs will only be set for new books." msgstr "" -#: config.py:427 +#: config.py:428 msgid "Default Update EPUB Cover when Updating EPUB?" msgstr "" -#: config.py:428 +#: config.py:429 msgid "On each download, FanFicFare offers an option to update the book cover image inside the EPUB from the web site when the EPUB is updated.
This sets whether that will default to on or off." msgstr "" -#: config.py:433 +#: config.py:434 msgid "Updating Calibre Options" msgstr "" -#: config.py:437 +#: config.py:438 msgid "Delete other existing formats?" msgstr "" -#: config.py:438 +#: config.py:439 msgid "" "Check this to automatically delete all other ebook formats when updating an existing book.\n" "Handy if you have both a Nook(epub) and Kindle(mobi), for example." msgstr "" -#: config.py:442 +#: config.py:443 msgid "Keep Existing Tags when Updating Metadata?" msgstr "" -#: config.py:443 +#: config.py:444 msgid "" "Existing tags will be kept and any new tags added.\n" "%(cmplt)s and %(inprog)s tags will be still be updated, if known.\n" @@ -169,336 +169,344 @@ msgid "" "(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)" msgstr "" -#: config.py:447 +#: config.py:448 msgid "Force Author into Author Sort?" msgstr "" -#: config.py:448 +#: config.py:449 msgid "" "If checked, the author(s) as given will be used for the Author Sort, too.\n" "If not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc." msgstr "" -#: config.py:452 +#: config.py:453 msgid "Force Title into Title Sort?" msgstr "" -#: config.py:453 +#: config.py:454 msgid "" "If checked, the title as given will be used for the Title Sort, too.\n" "If not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc." msgstr "" -#: config.py:457 +#: config.py:458 msgid "Check for existing Series Anthology books?" msgstr "" -#: config.py:458 +#: config.py:459 msgid "" "Check for existings Series Anthology books using each new story's series URL before downloading.\n" "Offer to skip downloading if a Series Anthology is found." msgstr "" -#: config.py:462 +#: config.py:463 msgid "Check for changed Story URL?" msgstr "" -#: config.py:463 +#: config.py:464 msgid "" "Warn you if an update will change the URL of an existing book.\n" "fanfiction.net URLs will change from http to https silently." msgstr "" -#: config.py:467 +#: config.py:468 msgid "Search EPUB text for Story URL?" msgstr "" -#: config.py:468 +#: config.py:469 msgid "" "Look for first valid story URL inside EPUB text if not found in metadata.\n" "Somewhat risky, could find wrong URL depending on EPUB content.\n" "Also finds and corrects bad ffnet URLs from ficsaver.com files." msgstr "" -#: config.py:472 +#: config.py:473 msgid "Mark added/updated books when finished?" msgstr "" -#: config.py:473 +#: config.py:474 msgid "" "Mark added/updated books when finished. Use with option below.\n" "You can also manually search for 'marked:fff_success'.\n" "'marked:fff_failed' is also available, or search 'marked:fff' for both." msgstr "" -#: config.py:477 +#: config.py:478 msgid "Show Marked books when finished?" msgstr "" -#: config.py:478 +#: config.py:479 msgid "" "Show Marked added/updated books only when finished.\n" "You can also manually search for 'marked:fff_success'.\n" "'marked:fff_failed' is also available, or search 'marked:fff' for both." msgstr "" -#: config.py:482 +#: config.py:483 msgid "Smarten Punctuation (EPUB only)" msgstr "" -#: config.py:483 +#: config.py:484 msgid "Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update." msgstr "" -#: config.py:487 +#: config.py:488 msgid "Automatically Convert new/update books?" msgstr "" -#: config.py:488 +#: config.py:489 msgid "" "Automatically call calibre's Convert for new/update books.\n" "Converts to the current output format as chosen in calibre's\n" "Preferences->Behavior settings." msgstr "" -#: config.py:492 +#: config.py:493 msgid "GUI Options" msgstr "" -#: config.py:496 +#: config.py:497 msgid "Take URLs from Clipboard?" msgstr "" -#: config.py:497 +#: config.py:498 msgid "Prefill URLs from valid URLs in Clipboard when Adding New." msgstr "" -#: config.py:501 +#: config.py:502 msgid "Default to Update when books selected?" msgstr "" -#: config.py:502 +#: config.py:503 msgid "" "The top FanFicFare plugin button will start Update if\n" "books are selected. If unchecked, it will always bring up 'Add New'." msgstr "" -#: config.py:506 +#: config.py:507 msgid "Keep 'Add New from URL(s)' dialog on top?" msgstr "" -#: config.py:507 +#: config.py:508 msgid "" "Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\n" "dialog on top of all other windows. Useful for dragging URLs onto it." msgstr "" -#: config.py:511 +#: config.py:512 msgid "Show estimated time left?" msgstr "" -#: config.py:512 +#: config.py:513 msgid "When a Progress Bar is shown, show a rough estimate of the time left." msgstr "" -#: config.py:516 +#: config.py:517 msgid "Misc Options" msgstr "" -#: config.py:520 +#: config.py:521 msgid "Inject calibre Series when none found?" msgstr "" -#: config.py:521 +#: config.py:522 msgid "" "If no series is found, inject the calibre series (if there is one) so \n" "it appears on the FanFicFare title page(not cover)." msgstr "" -#: config.py:525 +#: config.py:526 msgid "Search by Title/Author(s) for If Story Already Exists?" msgstr "" -#: config.py:526 +#: config.py:527 msgid "When checking If Story Already Exists FanFicFare will first match by URL Identifier. But if not found, it can also search existing books by Title and Author(s)." msgstr "" -#: config.py:530 +#: config.py:531 msgid "Reject List" msgstr "" -#: config.py:534 +#: config.py:535 msgid "Edit Reject URL List" msgstr "" -#: config.py:535 +#: config.py:536 msgid "Edit list of URLs FanFicFare will automatically Reject." msgstr "" -#: config.py:539 config.py:610 +#: config.py:540 config.py:611 msgid "Add Reject URLs" msgstr "" -#: config.py:540 +#: config.py:541 msgid "Add additional URLs to Reject as text." msgstr "" -#: config.py:544 +#: config.py:545 msgid "Edit Reject Reasons List" msgstr "" -#: config.py:545 config.py:600 +#: config.py:546 config.py:601 msgid "Customize the Reasons presented when Rejecting URLs" msgstr "" -#: config.py:549 +#: config.py:550 msgid "Reject Without Confirmation?" msgstr "" -#: config.py:550 +#: config.py:551 msgid "Always reject URLs on the Reject List without stopping and asking." msgstr "" -#: config.py:584 +#: config.py:585 msgid "Edit Reject URLs List" msgstr "" -#: config.py:598 +#: config.py:599 msgid "Reject Reasons" msgstr "" -#: config.py:599 +#: config.py:600 msgid "Customize Reject List Reasons" msgstr "" -#: config.py:608 +#: config.py:609 msgid "Reason why I rejected it" msgstr "" -#: config.py:608 +#: config.py:609 msgid "Title by Author" msgstr "" -#: config.py:611 +#: config.py:612 msgid "Add Reject URLs. Use: http://...,note or http://...,title by author - note
Invalid story URLs will be ignored." msgstr "" -#: config.py:612 +#: config.py:613 msgid "" "One URL per line:\n" "http://...,note\n" "http://...,title by author - note" msgstr "" -#: config.py:614 dialogs.py:1094 +#: config.py:615 dialogs.py:1094 msgid "Add this reason to all URLs added:" msgstr "" -#: config.py:630 +#: config.py:631 msgid "These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set %(isa)s and %(u)s/%(p)s for different sites." msgstr "" -#: config.py:635 +#: config.py:636 msgid "FanFicFare now includes find, color coding, and error checking for personal.ini editing. Red generally indicates errors." msgstr "" -#: config.py:654 config.py:695 config.py:696 +#: config.py:655 config.py:696 config.py:697 msgid "Edit personal.ini" msgstr "" -#: config.py:659 +#: config.py:660 msgid "Changes will only be saved if you click 'OK' to leave Customize FanFicFare." msgstr "" -#: config.py:663 +#: config.py:664 msgid "View Defaults" msgstr "" -#: config.py:664 +#: config.py:665 msgid "" "View all of the plugin's configurable settings\n" "and their default settings." msgstr "" -#: config.py:668 +#: config.py:669 msgid "Pass Calibre Columns into FanFicFare on Update/Overwrite" msgstr "" -#: config.py:669 +#: config.py:670 msgid "If checked, when updating/overwriting an existing book, FanFicFare will have the Calibre Columns available to use in replace_metadata, title_page, etc.
Click the button below to see the Calibre Column namess." msgstr "" -#: config.py:673 +#: config.py:674 msgid "Show Calibre Column Names" msgstr "" -#: config.py:674 +#: config.py:675 msgid "FanFicFare can pass the Calibre Columns into the download/update process.
This will show you the columns available by name." msgstr "" -#: config.py:685 +#: config.py:686 msgid "Plugin Defaults" msgstr "" -#: config.py:686 +#: config.py:687 msgid "Plugin Defaults (%s) (Read-Only)" msgstr "" -#: config.py:719 +#: config.py:720 msgid "Calibre Column Entry Names" msgstr "" -#: config.py:720 +#: config.py:721 msgid "Label (entry_name)" msgstr "" -#: config.py:740 +#: config.py:741 msgid "These settings provide integration with the %(rl)s Plugin. %(rl)s can automatically send to devices and change custom columns. You have to create and configure the lists in %(rl)s to be useful." msgstr "" -#: config.py:745 +#: config.py:746 msgid "Add new/updated stories to \"Send to Device\" Reading List(s)." msgstr "" -#: config.py:746 +#: config.py:747 msgid "Automatically add new/updated stories to these lists in the %(rl)s plugin." msgstr "" -#: config.py:751 +#: config.py:752 msgid "\"Send to Device\" Reading Lists" msgstr "" -#: config.py:752 config.py:755 config.py:769 config.py:772 +#: config.py:753 config.py:756 config.py:770 config.py:773 msgid "When enabled, new/updated stories will be automatically added to these lists." msgstr "" -#: config.py:762 +#: config.py:763 msgid "Add new/updated stories to \"To Read\" Reading List(s)." msgstr "" -#: config.py:763 +#: config.py:764 msgid "" "Automatically add new/updated stories to these lists in the %(rl)s plugin.\n" "Also offers menu option to remove stories from the \"To Read\" lists." msgstr "" -#: config.py:768 +#: config.py:769 msgid "\"To Read\" Reading Lists" msgstr "" -#: config.py:779 +#: config.py:780 msgid "Add stories back to \"Send to Device\" Reading List(s) when marked \"Read\"." msgstr "" -#: config.py:780 +#: config.py:781 msgid "Menu option to remove from \"To Read\" lists will also add stories back to \"Send to Device\" Reading List(s)" msgstr "" -#: config.py:808 +#: config.py:785 +msgid "Automatically run Remove \"New\" Chapter Marks when marking books \"Read\"." +msgstr "" + +#: config.py:786 +msgid "Menu option to remove from \"To Read\" lists will also remove \"(new)\" chapter marks created by personal.ini mark_new_chapters setting." +msgstr "" + +#: config.py:814 msgid "The Calibre cover image for a downloaded book can come from the story site(if EPUB and images are enabled), or from either Calibre's built-in random cover generator or the %(gc)s plugin." msgstr "" -#: config.py:816 +#: config.py:822 msgid "" "Update Calibre book cover image from EPUB when Calibre metadata is updated.\n" "Doesn't go looking for new images on 'Update Calibre Metadata Only'.\n" @@ -506,425 +514,425 @@ msgid "" "This comes before Generate Cover so %(gc)s(Plugin) use the image if configured to." msgstr "" -#: config.py:821 +#: config.py:827 msgid "Update Calibre Cover (from EPUB):" msgstr "" -#: config.py:839 +#: config.py:845 msgid "Generate a Calibre book cover image when Calibre metadata is updated.
Defaults to 'Yes, Always' for backward compatibility and because %(gc)s(Plugin) will only run if configured for Default or site." msgstr "" -#: config.py:843 +#: config.py:849 msgid "Generate Calibre Cover:" msgstr "" -#: config.py:870 +#: config.py:876 msgid "Plugin %(gc)s" msgstr "" -#: config.py:871 +#: config.py:877 msgid "Use plugin to create covers. Additional settings are below." msgstr "" -#: config.py:878 +#: config.py:884 msgid "Calibre Generate Cover" msgstr "" -#: config.py:879 +#: config.py:885 msgid "Call Calibre's Edit Metadata Generate cover feature to create a random cover each time a story is downloaded or updated.
Right click or long click the 'Generate cover' button in Calibre's Edit Metadata to customize." msgstr "" -#: config.py:893 +#: config.py:899 msgid "Generate Covers Only for New Books" msgstr "" -#: config.py:894 +#: config.py:900 msgid "Default is to generate a cover any time the calibre metadata is updated.
Used for both Calibre and Plugin generated covers." msgstr "" -#: config.py:900 +#: config.py:906 msgid "Inject/update the cover inside EPUB" msgstr "" -#: config.py:901 +#: config.py:907 msgid "Calibre's Polish feature will be used to inject or update the generated cover into the EPUB ebook file.
Used for both Calibre and Plugin generated covers." msgstr "" -#: config.py:907 +#: config.py:913 msgid "%(gc)s(Plugin) Settings" msgstr "" -#: config.py:915 +#: config.py:921 msgid "The %(gc)s plugin can create cover images for books using various metadata (including existing cover image). If you have %(gc)s installed, FanFicFare can run %(gc)s on new downloads and metadata updates. Pick a %(gc)s setting by site and/or one to use by Default." msgstr "" -#: config.py:933 config.py:937 config.py:950 +#: config.py:939 config.py:943 config.py:956 msgid "Default" msgstr "" -#: config.py:938 +#: config.py:944 msgid "On Metadata update, run %(gc)s with this setting, if there isn't a more specific setting below." msgstr "" -#: config.py:941 +#: config.py:947 msgid "On Metadata update, run %(gc)s with this setting for %(site)s stories." msgstr "" -#: config.py:964 +#: config.py:970 msgid "Allow %(gcset)s from %(pini)s to override" msgstr "" -#: config.py:965 +#: config.py:971 msgid "The %(pini)s parameter %(gcset)s allows you to choose a %(gc)s setting based on metadata rather than site, but it's much more complex.
%(gcset)s is ignored when this is off." msgstr "" -#: config.py:1003 +#: config.py:1009 msgid "These settings provide integration with the %(cp)s Plugin. %(cp)s can automatically update custom columns with page, word and reading level statistics. You have to create and configure the columns in %(cp)s first." msgstr "" -#: config.py:1008 +#: config.py:1014 msgid "If any of the settings below are checked, when stories are added or updated, the %(cp)s Plugin will be called to update the checked statistics." msgstr "" -#: config.py:1014 +#: config.py:1020 msgid "Which column and algorithm to use are configured in %(cp)s." msgstr "" -#: config.py:1024 +#: config.py:1030 msgid "Will overwrite word count from FanFicFare metadata if set to update the same custom column." msgstr "" -#: config.py:1029 +#: config.py:1035 msgid "Only run Count Page's Word Count if checked and FanFicFare metadata doesn't already have a word count. If this is used with one of the other Page Counts, the Page Count plugin will be called twice." msgstr "" -#: config.py:1065 +#: config.py:1071 msgid "These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFicFare confirmation dialogs back again." msgstr "" -#: config.py:1070 +#: config.py:1076 msgid "Keyboard shortcuts..." msgstr "" -#: config.py:1071 +#: config.py:1077 msgid "Edit the keyboard shortcuts associated with this plugin" msgstr "" -#: config.py:1075 +#: config.py:1081 msgid "Reset disabled &confirmation dialogs" msgstr "" -#: config.py:1076 +#: config.py:1082 msgid "Reset all show me again dialogs for the FanFicFare plugin" msgstr "" -#: config.py:1080 +#: config.py:1086 msgid "&View library preferences..." msgstr "" -#: config.py:1081 +#: config.py:1087 msgid "View data stored in the library database for this plugin" msgstr "" -#: config.py:1092 +#: config.py:1098 msgid "Done" msgstr "" -#: config.py:1093 +#: config.py:1099 msgid "Confirmation dialogs have all been reset" msgstr "" -#: config.py:1141 +#: config.py:1147 msgid "Category" msgstr "" -#: config.py:1142 +#: config.py:1148 msgid "Genre" msgstr "" -#: config.py:1143 +#: config.py:1149 msgid "Language" msgstr "" -#: config.py:1144 fff_plugin.py:1303 fff_plugin.py:1501 fff_plugin.py:1531 +#: config.py:1150 fff_plugin.py:1376 fff_plugin.py:1574 fff_plugin.py:1604 msgid "Status" msgstr "" -#: config.py:1145 +#: config.py:1151 msgid "Status:%(cmplt)s" msgstr "" -#: config.py:1146 +#: config.py:1152 msgid "Status:%(inprog)s" msgstr "" -#: config.py:1147 config.py:1295 +#: config.py:1153 config.py:1301 msgid "Series" msgstr "" -#: config.py:1148 +#: config.py:1154 msgid "Characters" msgstr "" -#: config.py:1149 +#: config.py:1155 msgid "Relationships" msgstr "" -#: config.py:1150 +#: config.py:1156 msgid "Published" msgstr "" -#: config.py:1151 fff_plugin.py:1614 fff_plugin.py:1633 +#: config.py:1157 fff_plugin.py:1687 fff_plugin.py:1706 msgid "Updated" msgstr "" -#: config.py:1152 +#: config.py:1158 msgid "Created" msgstr "" -#: config.py:1153 +#: config.py:1159 msgid "Rating" msgstr "" -#: config.py:1154 +#: config.py:1160 msgid "Warnings" msgstr "" -#: config.py:1155 +#: config.py:1161 msgid "Chapters" msgstr "" -#: config.py:1156 +#: config.py:1162 msgid "Words" msgstr "" -#: config.py:1157 +#: config.py:1163 msgid "Site" msgstr "" -#: config.py:1158 +#: config.py:1164 msgid "Story ID" msgstr "" -#: config.py:1159 +#: config.py:1165 msgid "Author ID" msgstr "" -#: config.py:1160 +#: config.py:1166 msgid "Extra Tags" msgstr "" -#: config.py:1161 config.py:1287 dialogs.py:885 dialogs.py:981 -#: fff_plugin.py:1303 fff_plugin.py:1501 fff_plugin.py:1531 +#: config.py:1167 config.py:1293 dialogs.py:885 dialogs.py:981 +#: fff_plugin.py:1376 fff_plugin.py:1574 fff_plugin.py:1604 msgid "Title" msgstr "" -#: config.py:1162 +#: config.py:1168 msgid "Story URL" msgstr "" -#: config.py:1163 +#: config.py:1169 msgid "Description" msgstr "" -#: config.py:1164 dialogs.py:885 dialogs.py:981 fff_plugin.py:1303 -#: fff_plugin.py:1501 fff_plugin.py:1531 +#: config.py:1170 dialogs.py:885 dialogs.py:981 fff_plugin.py:1376 +#: fff_plugin.py:1574 fff_plugin.py:1604 msgid "Author" msgstr "" -#: config.py:1165 +#: config.py:1171 msgid "Author URL" msgstr "" -#: config.py:1166 +#: config.py:1172 msgid "File Format" msgstr "" -#: config.py:1167 +#: config.py:1173 msgid "File Extension" msgstr "" -#: config.py:1168 +#: config.py:1174 msgid "Site Abbrev" msgstr "" -#: config.py:1169 +#: config.py:1175 msgid "FanFicFare Version" msgstr "" -#: config.py:1184 +#: config.py:1190 msgid "If you have custom columns defined, they will be listed below. Choose a metadata value type to fill your columns automatically." msgstr "" -#: config.py:1209 +#: config.py:1215 msgid "Update this %s column(%s) with..." msgstr "" -#: config.py:1219 +#: config.py:1225 msgid "Values that aren't valid for this enumeration column will be ignored." msgstr "" -#: config.py:1219 config.py:1221 +#: config.py:1225 config.py:1227 msgid "Metadata values valid for this type of column." msgstr "" -#: config.py:1224 config.py:1314 +#: config.py:1230 config.py:1320 msgid "New Only" msgstr "" -#: config.py:1225 +#: config.py:1231 msgid "" "Write to %s(%s) only for new\n" "books, not updates to existing books." msgstr "" -#: config.py:1236 +#: config.py:1242 msgid "Allow %(ccset)s from %(pini)s to override" msgstr "" -#: config.py:1237 +#: config.py:1243 msgid "The %(pini)s parameter %(ccset)s allows you to set custom columns to site specific values that aren't common to all sites.
%(ccset)s is ignored when this is off." msgstr "" -#: config.py:1241 +#: config.py:1247 msgid "Special column:" msgstr "" -#: config.py:1246 +#: config.py:1252 msgid "Update/Overwrite Error Column:" msgstr "" -#: config.py:1247 +#: config.py:1253 msgid "" "When an update or overwrite of an existing story fails, record the reason in this column.\n" "(Text and Long Text columns only.)" msgstr "" -#: config.py:1261 +#: config.py:1267 msgid "Saved Metadata Column:" msgstr "" -#: config.py:1262 +#: config.py:1268 msgid "If set, FanFicFare will save a copy of all its metadata in this column when the book is downloaded or updated.
The metadata from this column can later be used to update custom columns without having to request the metadata from the server again.
(Long Text columns only.)" msgstr "" -#: config.py:1288 +#: config.py:1294 msgid "Author(s)" msgstr "" -#: config.py:1289 +#: config.py:1295 msgid "Publisher" msgstr "" -#: config.py:1290 +#: config.py:1296 msgid "Tags" msgstr "" -#: config.py:1291 +#: config.py:1297 msgid "Languages" msgstr "" -#: config.py:1292 +#: config.py:1298 msgid "Published Date" msgstr "" -#: config.py:1293 +#: config.py:1299 msgid "Date" msgstr "" -#: config.py:1294 +#: config.py:1300 msgid "Comments" msgstr "" -#: config.py:1296 +#: config.py:1302 msgid "Ids(url id only)" msgstr "" -#: config.py:1301 +#: config.py:1307 msgid "The standard calibre metadata columns are listed below. You may choose whether FanFicFare will fill each column automatically on updates or only for new books." msgstr "" -#: config.py:1315 +#: config.py:1321 msgid "" "Write to %s only for new\n" "books, not updates to existing books." msgstr "" -#: config.py:1324 +#: config.py:1330 msgid "Other Standard Column Options" msgstr "" -#: config.py:1329 +#: config.py:1335 msgid "Set Calibre Author URL" msgstr "" -#: config.py:1330 +#: config.py:1336 msgid "Set Calibre Author URL to Author's URL on story site." msgstr "" -#: config.py:1347 +#: config.py:1353 msgid "These settings will allow FanFicFare to fetch story URLs from your email account. It will only look for story URLs in unread emails in the folder specified below." msgstr "" -#: config.py:1352 +#: config.py:1358 msgid "IMAP Server Name" msgstr "" -#: config.py:1353 +#: config.py:1359 msgid "Name of IMAP server--must allow IMAP4 with SSL. Eg: imap.gmail.com" msgstr "" -#: config.py:1362 +#: config.py:1368 msgid "IMAP User Name" msgstr "" -#: config.py:1363 +#: config.py:1369 msgid "" "Name of IMAP user. Eg: yourname@gmail.com\n" "Note that Gmail accounts need to have IMAP enabled in Gmail Settings first." msgstr "" -#: config.py:1372 +#: config.py:1378 msgid "IMAP User Password" msgstr "" -#: config.py:1373 +#: config.py:1379 msgid "IMAP password. If left empty, FanFicFare will ask you for your password when you use the feature." msgstr "" -#: config.py:1383 +#: config.py:1389 msgid "Remember Password for Session (when not saved above)" msgstr "" -#: config.py:1384 +#: config.py:1390 msgid "If checked, and no password is entered above, FanFicFare will remember your password until you close calibre or change Libraries." msgstr "" -#: config.py:1389 +#: config.py:1395 msgid "IMAP Folder Name" msgstr "" -#: config.py:1390 +#: config.py:1396 msgid "Name of IMAP folder to search for new emails. The folder (or label) has to already exist. Use INBOX for your default inbox." msgstr "" -#: config.py:1399 +#: config.py:1405 msgid "Mark Emails Read" msgstr "" -#: config.py:1400 +#: config.py:1406 msgid "If checked, emails will be marked as having been read if they contain any story URLs." msgstr "" -#: config.py:1405 +#: config.py:1411 msgid "Discard URLs on Reject List" msgstr "" -#: config.py:1406 +#: config.py:1412 msgid "If checked, FanFicFare will silently discard story URLs from emails that are on your Reject URL List.
Otherwise they will appear and you will see the normal Reject URL dialog.
The Emails will still be marked Read if configured to." msgstr "" -#: config.py:1411 +#: config.py:1417 msgid "It's safest if you create a separate email account that you use only for your story update notices. FanFicFare and calibre cannot guarantee that malicious code cannot get your email password once you've entered it.
Use this feature at your own risk. " msgstr "" @@ -1129,7 +1137,7 @@ msgstr "" msgid "less than 1 second" msgstr "" -#: dialogs.py:713 fff_plugin.py:358 fff_plugin.py:361 +#: dialogs.py:713 fff_plugin.py:365 fff_plugin.py:368 msgid "About FanFicFare" msgstr "" @@ -1145,7 +1153,7 @@ msgstr "" msgid "What sort of update to perform. May set default from plugin configuration." msgstr "" -#: dialogs.py:885 fff_plugin.py:1303 fff_plugin.py:1501 fff_plugin.py:1531 +#: dialogs.py:885 fff_plugin.py:1376 fff_plugin.py:1574 fff_plugin.py:1604 msgid "Comment" msgstr "" @@ -1241,603 +1249,623 @@ msgstr "" msgid "Enter Email Password for %s:" msgstr "" -#: fff_plugin.py:114 fff_plugin.py:145 +#: fff_plugin.py:115 fff_plugin.py:146 msgid "FanFicFare" msgstr "" -#: fff_plugin.py:115 +#: fff_plugin.py:116 msgid "Download FanFiction stories from various web sites" msgstr "" -#: fff_plugin.py:276 +#: fff_plugin.py:277 msgid "&Download from URLs" msgstr "" -#: fff_plugin.py:278 +#: fff_plugin.py:279 msgid "Download FanFiction Books from URLs" msgstr "" -#: fff_plugin.py:281 +#: fff_plugin.py:282 msgid "&Update Existing FanFiction Books" msgstr "" -#: fff_plugin.py:286 +#: fff_plugin.py:287 msgid "Get Story URLs from &Email" msgstr "" -#: fff_plugin.py:290 fff_plugin.py:478 +#: fff_plugin.py:291 fff_plugin.py:487 msgid "Get Story URLs from Web Page" msgstr "" -#: fff_plugin.py:296 +#: fff_plugin.py:297 msgid "&Make Anthology Epub from URLs" msgstr "" -#: fff_plugin.py:298 +#: fff_plugin.py:299 msgid "Make FanFiction Anthology Epub from URLs" msgstr "" -#: fff_plugin.py:301 +#: fff_plugin.py:302 msgid "Make Anthology Epub from Web Page" msgstr "" -#: fff_plugin.py:303 +#: fff_plugin.py:304 msgid "Make FanFiction Anthology Epub from Web Page" msgstr "" -#: fff_plugin.py:306 +#: fff_plugin.py:307 msgid "Update Anthology Epub" msgstr "" -#: fff_plugin.py:308 +#: fff_plugin.py:309 msgid "Update FanFiction Anthology Epub" msgstr "" -#: fff_plugin.py:315 +#: fff_plugin.py:316 msgid "Mark Unread: Add to \"To Read\" and \"Send to Device\" Lists" msgstr "" -#: fff_plugin.py:317 +#: fff_plugin.py:318 msgid "Mark Read: Remove from \"To Read\" and add to \"Send to Device\" Lists" msgstr "" -#: fff_plugin.py:319 fff_plugin.py:324 +#: fff_plugin.py:320 fff_plugin.py:325 msgid "Mark Read: Remove from \"To Read\" Lists" msgstr "" -#: fff_plugin.py:321 +#: fff_plugin.py:322 msgid "Add to \"Send to Device\" Lists" msgstr "" -#: fff_plugin.py:323 +#: fff_plugin.py:324 msgid "Mark Unread: Add to \"To Read\" Lists" msgstr "" -#: fff_plugin.py:339 +#: fff_plugin.py:340 +msgid "Remove \"New\" Chapter Marks from Selected books" +msgstr "" + +#: fff_plugin.py:346 msgid "Get Story URLs from Selected Books" msgstr "" -#: fff_plugin.py:344 +#: fff_plugin.py:351 msgid "Reject Selected Books" msgstr "" -#: fff_plugin.py:352 +#: fff_plugin.py:359 msgid "&Configure FanFicFare" msgstr "" -#: fff_plugin.py:355 +#: fff_plugin.py:362 msgid "Configure FanFicFare" msgstr "" -#: fff_plugin.py:410 +#: fff_plugin.py:417 msgid "Cannot Update Reading Lists from Device View" msgstr "" -#: fff_plugin.py:414 +#: fff_plugin.py:421 msgid "No Selected Books to Update Reading Lists" msgstr "" -#: fff_plugin.py:422 +#: fff_plugin.py:431 msgid "FanFicFare Email Settings are not configured." msgstr "" -#: fff_plugin.py:442 +#: fff_plugin.py:451 msgid "Fetching Story URLs from Email..." msgstr "" -#: fff_plugin.py:454 +#: fff_plugin.py:463 msgid "Finished Fetching Story URLs from Email." msgstr "" -#: fff_plugin.py:461 +#: fff_plugin.py:470 msgid "No Valid Story URLs Found in Unread Emails." msgstr "" -#: fff_plugin.py:463 +#: fff_plugin.py:472 msgid "(%d Story URLs Skipped, on Rejected URL List)" msgstr "" -#: fff_plugin.py:464 +#: fff_plugin.py:473 msgid "Get Story URLs from Email" msgstr "" -#: fff_plugin.py:487 +#: fff_plugin.py:496 msgid "Fetching Story URLs from Page..." msgstr "" -#: fff_plugin.py:491 +#: fff_plugin.py:500 msgid "Finished Fetching Story URLs from Page." msgstr "" -#: fff_plugin.py:497 fff_plugin.py:549 +#: fff_plugin.py:506 fff_plugin.py:558 msgid "List of Story URLs" msgstr "" -#: fff_plugin.py:498 +#: fff_plugin.py:507 msgid "No Valid Story URLs found on given page." msgstr "" -#: fff_plugin.py:513 +#: fff_plugin.py:522 fff_plugin.py:575 msgid "No Selected Books to Get URLs From" msgstr "" -#: fff_plugin.py:531 +#: fff_plugin.py:540 msgid "Collecting URLs for stories..." msgstr "" -#: fff_plugin.py:532 +#: fff_plugin.py:541 msgid "Get URLs for stories" msgstr "" -#: fff_plugin.py:533 fff_plugin.py:580 fff_plugin.py:773 +#: fff_plugin.py:542 fff_plugin.py:653 fff_plugin.py:846 msgid "URL retrieved" msgstr "" -#: fff_plugin.py:553 +#: fff_plugin.py:562 msgid "List of URLs" msgstr "" -#: fff_plugin.py:554 +#: fff_plugin.py:563 msgid "No Story URLs found in selected books." msgstr "" #: fff_plugin.py:570 -msgid "No Selected Books have URLs to Reject" +msgid "Can only UnNew books in library" msgstr "" -#: fff_plugin.py:578 -msgid "Collecting URLs for Reject List..." +#: fff_plugin.py:587 +msgid "UnNewing books..." msgstr "" -#: fff_plugin.py:579 -msgid "Get URLs for Reject List" +#: fff_plugin.py:588 +msgid "UnNew Books" msgstr "" -#: fff_plugin.py:614 -msgid "Proceed to Remove?" +#: fff_plugin.py:589 +msgid "Books UnNewed" msgstr "" -#: fff_plugin.py:614 -msgid "Rejecting FanFicFare URLs: None of the books selected have FanFiction URLs." -msgstr "" - -#: fff_plugin.py:636 -msgid "Cannot Make Anthologys without %s" -msgstr "" - -#: fff_plugin.py:640 fff_plugin.py:750 -msgid "Cannot Update Books from Device View" -msgstr "" - -#: fff_plugin.py:644 -msgid "Can only update 1 anthology at a time" -msgstr "" - -#: fff_plugin.py:653 -msgid "Can only Update Epub Anthologies" -msgstr "" - -#: fff_plugin.py:671 fff_plugin.py:672 -msgid "Cannot Update Anthology" -msgstr "" - -#: fff_plugin.py:672 -msgid "Book isn't an FanFicFare Anthology or contains book(s) without valid Story URLs." -msgstr "" - -#: fff_plugin.py:679 -msgid "Fetching Story URLs for Series..." -msgstr "" - -#: fff_plugin.py:689 -msgid "Finished Fetching Story URLs for Series." -msgstr "" - -#: fff_plugin.py:736 -msgid "There are %d stories in the current anthology that are not going to be kept if you go ahead." -msgstr "" - -#: fff_plugin.py:737 -msgid "Story URLs that will be removed:" -msgstr "" - -#: fff_plugin.py:739 -msgid "Update anyway?" -msgstr "" - -#: fff_plugin.py:740 -msgid "Stories Removed" -msgstr "" - -#: fff_plugin.py:757 -msgid "No Selected Books to Update" -msgstr "" - -#: fff_plugin.py:771 -msgid "Collecting stories for update..." -msgstr "" - -#: fff_plugin.py:772 -msgid "Get stories for updates" -msgstr "" - -#: fff_plugin.py:782 -msgid "Update Existing List" -msgstr "" - -#: fff_plugin.py:841 -msgid "Started fetching metadata for %s stories." -msgstr "" - -#: fff_plugin.py:847 -msgid "No valid story URLs entered." -msgstr "" - -#: fff_plugin.py:872 fff_plugin.py:878 -msgid "Reject URL?" -msgstr "" - -#: fff_plugin.py:879 fff_plugin.py:897 -msgid "%s is on your Reject URL list:" -msgstr "" - -#: fff_plugin.py:881 -msgid "Click 'Yes' to Reject." -msgstr "" - -#: fff_plugin.py:882 fff_plugin.py:1009 -msgid "Click 'No' to download anyway." -msgstr "" - -#: fff_plugin.py:884 -msgid "Story on Reject URLs list (%s)." -msgstr "" - -#: fff_plugin.py:887 -msgid "Rejected" -msgstr "" - -#: fff_plugin.py:890 -msgid "Remove Reject URL?" -msgstr "" - -#: fff_plugin.py:896 -msgid "Remove URL from Reject List?" -msgstr "" - -#: fff_plugin.py:899 -msgid "Click 'Yes' to remove it from the list," -msgstr "" - -#: fff_plugin.py:900 -msgid "Click 'No' to leave it on the list." -msgstr "" - -#: fff_plugin.py:917 -msgid "Cannot update non-epub format." -msgstr "" - -#: fff_plugin.py:987 -msgid "Are You an Adult?" -msgstr "" - -#: fff_plugin.py:988 -msgid "%s requires that you be an adult. Please confirm you are an adult in your locale:" -msgstr "" - -#: fff_plugin.py:1000 -msgid "Skip Story?" -msgstr "" - -#: fff_plugin.py:1006 -msgid "Skip Anthology Story?" -msgstr "" - -#: fff_plugin.py:1007 -msgid "\"%s\" is in series \"%s\" that you have an anthology book for." -msgstr "" - -#: fff_plugin.py:1008 -msgid "Click 'Yes' to Skip." -msgstr "" - -#: fff_plugin.py:1011 -msgid "Story in Series Anthology(%s)." -msgstr "" - -#: fff_plugin.py:1016 -msgid "Skipped" -msgstr "" - -#: fff_plugin.py:1046 -msgid "Add" -msgstr "" - -#: fff_plugin.py:1059 -msgid "Meta" -msgstr "" - -#: fff_plugin.py:1090 -msgid "Skipping duplicate story." -msgstr "" - -#: fff_plugin.py:1093 -msgid "More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite." -msgstr "" - -#: fff_plugin.py:1104 -msgid "Update" -msgstr "" - -#: fff_plugin.py:1112 fff_plugin.py:1119 -msgid "Change Story URL?" -msgstr "" - -#: fff_plugin.py:1120 -msgid "%s by %s is already in your library with a different source URL:" -msgstr "" - -#: fff_plugin.py:1121 -msgid "In library: %(liburl)s" -msgstr "" - -#: fff_plugin.py:1122 fff_plugin.py:1136 -msgid "New URL: %(newurl)s" -msgstr "" - -#: fff_plugin.py:1123 -msgid "Click 'Yes' to update/overwrite book with new URL." -msgstr "" - -#: fff_plugin.py:1124 -msgid "Click 'No' to skip updating/overwriting this book." -msgstr "" - -#: fff_plugin.py:1126 fff_plugin.py:1133 -msgid "Download as New Book?" -msgstr "" - -#: fff_plugin.py:1134 -msgid "%s by %s is already in your library with a different source URL." -msgstr "" - -#: fff_plugin.py:1135 -msgid "You chose not to update the existing book. Do you want to add a new book for this URL?" -msgstr "" - -#: fff_plugin.py:1137 -msgid "Click 'Yes' to a new book with new URL." -msgstr "" - -#: fff_plugin.py:1138 -msgid "Click 'No' to skip URL." -msgstr "" - -#: fff_plugin.py:1144 -msgid "Update declined by user due to differing story URL(%s)" -msgstr "" - -#: fff_plugin.py:1147 -msgid "Different URL" -msgstr "" - -#: fff_plugin.py:1152 -msgid "Metadata collected." -msgstr "" - -#: fff_plugin.py:1168 -msgid "Already contains %d chapters." -msgstr "" - -#: fff_plugin.py:1170 jobs.py:209 -msgid "Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." -msgstr "" - -#: fff_plugin.py:1172 -msgid "FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update." -msgstr "" - -#: fff_plugin.py:1184 -msgid "Not Overwriting, web site is not newer." -msgstr "" - -#: fff_plugin.py:1299 -msgid "None of the %d URLs/stories given can be/need to be downloaded." -msgstr "" - -#: fff_plugin.py:1300 fff_plugin.py:1497 fff_plugin.py:1527 -msgid "See log for details." -msgstr "" - -#: fff_plugin.py:1301 -msgid "Proceed with updating your library(Error Column, if configured)?" -msgstr "" - -#: fff_plugin.py:1308 fff_plugin.py:1509 -msgid "Bad" -msgstr "" - -#: fff_plugin.py:1316 -msgid "FanFicFare download ended" -msgstr "" - -#: fff_plugin.py:1316 fff_plugin.py:1552 -msgid "FanFicFare log" -msgstr "" - -#: fff_plugin.py:1336 -msgid "Download FanFiction Book" -msgstr "" - -#: fff_plugin.py:1343 -msgid "Starting %d FanFicFare Downloads" -msgstr "" - -#: fff_plugin.py:1373 -msgid "Story Details:" -msgstr "" - -#: fff_plugin.py:1376 -msgid "Error Updating Metadata" -msgstr "" - -#: fff_plugin.py:1377 -msgid "An error has occurred while FanFicFare was updating calibre's metadata for %s." -msgstr "" - -#: fff_plugin.py:1378 -msgid "The ebook has been updated, but the metadata has not." -msgstr "" - -#: fff_plugin.py:1430 -msgid "Finished Adding/Updating %d books." -msgstr "" - -#: fff_plugin.py:1460 +#: fff_plugin.py:628 fff_plugin.py:1533 msgid "Starting auto conversion of %d books." msgstr "" -#: fff_plugin.py:1481 +#: fff_plugin.py:643 +msgid "No Selected Books have URLs to Reject" +msgstr "" + +#: fff_plugin.py:651 +msgid "Collecting URLs for Reject List..." +msgstr "" + +#: fff_plugin.py:652 +msgid "Get URLs for Reject List" +msgstr "" + +#: fff_plugin.py:687 +msgid "Proceed to Remove?" +msgstr "" + +#: fff_plugin.py:687 +msgid "Rejecting FanFicFare URLs: None of the books selected have FanFiction URLs." +msgstr "" + +#: fff_plugin.py:709 +msgid "Cannot Make Anthologys without %s" +msgstr "" + +#: fff_plugin.py:713 fff_plugin.py:823 +msgid "Cannot Update Books from Device View" +msgstr "" + +#: fff_plugin.py:717 +msgid "Can only update 1 anthology at a time" +msgstr "" + +#: fff_plugin.py:726 +msgid "Can only Update Epub Anthologies" +msgstr "" + +#: fff_plugin.py:744 fff_plugin.py:745 +msgid "Cannot Update Anthology" +msgstr "" + +#: fff_plugin.py:745 +msgid "Book isn't an FanFicFare Anthology or contains book(s) without valid Story URLs." +msgstr "" + +#: fff_plugin.py:752 +msgid "Fetching Story URLs for Series..." +msgstr "" + +#: fff_plugin.py:762 +msgid "Finished Fetching Story URLs for Series." +msgstr "" + +#: fff_plugin.py:809 +msgid "There are %d stories in the current anthology that are not going to be kept if you go ahead." +msgstr "" + +#: fff_plugin.py:810 +msgid "Story URLs that will be removed:" +msgstr "" + +#: fff_plugin.py:812 +msgid "Update anyway?" +msgstr "" + +#: fff_plugin.py:813 +msgid "Stories Removed" +msgstr "" + +#: fff_plugin.py:830 +msgid "No Selected Books to Update" +msgstr "" + +#: fff_plugin.py:844 +msgid "Collecting stories for update..." +msgstr "" + +#: fff_plugin.py:845 +msgid "Get stories for updates" +msgstr "" + +#: fff_plugin.py:855 +msgid "Update Existing List" +msgstr "" + +#: fff_plugin.py:914 +msgid "Started fetching metadata for %s stories." +msgstr "" + +#: fff_plugin.py:920 +msgid "No valid story URLs entered." +msgstr "" + +#: fff_plugin.py:945 fff_plugin.py:951 +msgid "Reject URL?" +msgstr "" + +#: fff_plugin.py:952 fff_plugin.py:970 +msgid "%s is on your Reject URL list:" +msgstr "" + +#: fff_plugin.py:954 +msgid "Click 'Yes' to Reject." +msgstr "" + +#: fff_plugin.py:955 fff_plugin.py:1082 +msgid "Click 'No' to download anyway." +msgstr "" + +#: fff_plugin.py:957 +msgid "Story on Reject URLs list (%s)." +msgstr "" + +#: fff_plugin.py:960 +msgid "Rejected" +msgstr "" + +#: fff_plugin.py:963 +msgid "Remove Reject URL?" +msgstr "" + +#: fff_plugin.py:969 +msgid "Remove URL from Reject List?" +msgstr "" + +#: fff_plugin.py:972 +msgid "Click 'Yes' to remove it from the list," +msgstr "" + +#: fff_plugin.py:973 +msgid "Click 'No' to leave it on the list." +msgstr "" + +#: fff_plugin.py:990 +msgid "Cannot update non-epub format." +msgstr "" + +#: fff_plugin.py:1060 +msgid "Are You an Adult?" +msgstr "" + +#: fff_plugin.py:1061 +msgid "%s requires that you be an adult. Please confirm you are an adult in your locale:" +msgstr "" + +#: fff_plugin.py:1073 +msgid "Skip Story?" +msgstr "" + +#: fff_plugin.py:1079 +msgid "Skip Anthology Story?" +msgstr "" + +#: fff_plugin.py:1080 +msgid "\"%s\" is in series \"%s\" that you have an anthology book for." +msgstr "" + +#: fff_plugin.py:1081 +msgid "Click 'Yes' to Skip." +msgstr "" + +#: fff_plugin.py:1084 +msgid "Story in Series Anthology(%s)." +msgstr "" + +#: fff_plugin.py:1089 +msgid "Skipped" +msgstr "" + +#: fff_plugin.py:1119 +msgid "Add" +msgstr "" + +#: fff_plugin.py:1132 +msgid "Meta" +msgstr "" + +#: fff_plugin.py:1163 +msgid "Skipping duplicate story." +msgstr "" + +#: fff_plugin.py:1166 +msgid "More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite." +msgstr "" + +#: fff_plugin.py:1177 +msgid "Update" +msgstr "" + +#: fff_plugin.py:1185 fff_plugin.py:1192 +msgid "Change Story URL?" +msgstr "" + +#: fff_plugin.py:1193 +msgid "%s by %s is already in your library with a different source URL:" +msgstr "" + +#: fff_plugin.py:1194 +msgid "In library: %(liburl)s" +msgstr "" + +#: fff_plugin.py:1195 fff_plugin.py:1209 +msgid "New URL: %(newurl)s" +msgstr "" + +#: fff_plugin.py:1196 +msgid "Click 'Yes' to update/overwrite book with new URL." +msgstr "" + +#: fff_plugin.py:1197 +msgid "Click 'No' to skip updating/overwriting this book." +msgstr "" + +#: fff_plugin.py:1199 fff_plugin.py:1206 +msgid "Download as New Book?" +msgstr "" + +#: fff_plugin.py:1207 +msgid "%s by %s is already in your library with a different source URL." +msgstr "" + +#: fff_plugin.py:1208 +msgid "You chose not to update the existing book. Do you want to add a new book for this URL?" +msgstr "" + +#: fff_plugin.py:1210 +msgid "Click 'Yes' to a new book with new URL." +msgstr "" + +#: fff_plugin.py:1211 +msgid "Click 'No' to skip URL." +msgstr "" + +#: fff_plugin.py:1217 +msgid "Update declined by user due to differing story URL(%s)" +msgstr "" + +#: fff_plugin.py:1220 +msgid "Different URL" +msgstr "" + +#: fff_plugin.py:1225 +msgid "Metadata collected." +msgstr "" + +#: fff_plugin.py:1241 +msgid "Already contains %d chapters." +msgstr "" + +#: fff_plugin.py:1243 jobs.py:211 +msgid "Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." +msgstr "" + +#: fff_plugin.py:1245 +msgid "FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update." +msgstr "" + +#: fff_plugin.py:1257 +msgid "Not Overwriting, web site is not newer." +msgstr "" + +#: fff_plugin.py:1372 +msgid "None of the %d URLs/stories given can be/need to be downloaded." +msgstr "" + +#: fff_plugin.py:1373 fff_plugin.py:1570 fff_plugin.py:1600 +msgid "See log for details." +msgstr "" + +#: fff_plugin.py:1374 +msgid "Proceed with updating your library(Error Column, if configured)?" +msgstr "" + +#: fff_plugin.py:1381 fff_plugin.py:1582 +msgid "Bad" +msgstr "" + +#: fff_plugin.py:1389 +msgid "FanFicFare download ended" +msgstr "" + +#: fff_plugin.py:1389 fff_plugin.py:1625 +msgid "FanFicFare log" +msgstr "" + +#: fff_plugin.py:1409 +msgid "Download FanFiction Book" +msgstr "" + +#: fff_plugin.py:1416 +msgid "Starting %d FanFicFare Downloads" +msgstr "" + +#: fff_plugin.py:1446 +msgid "Story Details:" +msgstr "" + +#: fff_plugin.py:1449 +msgid "Error Updating Metadata" +msgstr "" + +#: fff_plugin.py:1450 +msgid "An error has occurred while FanFicFare was updating calibre's metadata for %s." +msgstr "" + +#: fff_plugin.py:1451 +msgid "The ebook has been updated, but the metadata has not." +msgstr "" + +#: fff_plugin.py:1503 +msgid "Finished Adding/Updating %d books." +msgstr "" + +#: fff_plugin.py:1554 msgid "No Good Stories for Anthology" msgstr "" -#: fff_plugin.py:1482 +#: fff_plugin.py:1555 msgid "No good stories/updates where downloaded, Anthology creation/update aborted." msgstr "" -#: fff_plugin.py:1487 fff_plugin.py:1526 +#: fff_plugin.py:1560 fff_plugin.py:1599 msgid "FanFicFare found %s good and %s bad updates." msgstr "" -#: fff_plugin.py:1494 +#: fff_plugin.py:1567 msgid "Are you sure you want to continue with creating/updating this Anthology?" msgstr "" -#: fff_plugin.py:1495 +#: fff_plugin.py:1568 msgid "Any updates that failed will not be included in the Anthology." msgstr "" -#: fff_plugin.py:1496 +#: fff_plugin.py:1569 msgid "However, if there's an older version, it will still be included." msgstr "" -#: fff_plugin.py:1499 +#: fff_plugin.py:1572 msgid "Proceed with updating this anthology and your library?" msgstr "" -#: fff_plugin.py:1507 +#: fff_plugin.py:1580 msgid "Good" msgstr "" -#: fff_plugin.py:1528 +#: fff_plugin.py:1601 msgid "Proceed with updating your library?" msgstr "" -#: fff_plugin.py:1552 +#: fff_plugin.py:1625 msgid "FanFicFare download complete" msgstr "" -#: fff_plugin.py:1565 +#: fff_plugin.py:1638 msgid "Merging %s books." msgstr "" -#: fff_plugin.py:1605 +#: fff_plugin.py:1678 msgid "FanFicFare Adding/Updating books." msgstr "" -#: fff_plugin.py:1612 +#: fff_plugin.py:1685 msgid "Updating calibre for FanFiction stories..." msgstr "" -#: fff_plugin.py:1613 +#: fff_plugin.py:1686 msgid "Update calibre for FanFiction stories" msgstr "" -#: fff_plugin.py:1622 +#: fff_plugin.py:1695 msgid "Adding/Updating %s BAD books." msgstr "" -#: fff_plugin.py:1631 +#: fff_plugin.py:1704 msgid "Updating calibre for BAD FanFiction stories..." msgstr "" -#: fff_plugin.py:1632 +#: fff_plugin.py:1705 msgid "Update calibre for BAD FanFiction stories" msgstr "" -#: fff_plugin.py:1658 +#: fff_plugin.py:1731 msgid "Adding format to book failed for some reason..." msgstr "" -#: fff_plugin.py:1661 +#: fff_plugin.py:1734 msgid "Error" msgstr "" -#: fff_plugin.py:1974 +#: fff_plugin.py:2047 msgid "You configured FanFicFare to automatically update Reading Lists, but you don't have the %s plugin installed anymore?" msgstr "" -#: fff_plugin.py:1986 +#: fff_plugin.py:2059 msgid "You configured FanFicFare to automatically update \"To Read\" Reading Lists, but you don't have any lists set?" msgstr "" -#: fff_plugin.py:1996 fff_plugin.py:2014 +#: fff_plugin.py:2069 fff_plugin.py:2087 msgid "You configured FanFicFare to automatically update Reading List '%s', but you don't have a list of that name?" msgstr "" -#: fff_plugin.py:2002 +#: fff_plugin.py:2075 msgid "You configured FanFicFare to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?" msgstr "" -#: fff_plugin.py:2123 +#: fff_plugin.py:2196 msgid "No story URL found." msgstr "" -#: fff_plugin.py:2126 +#: fff_plugin.py:2199 msgid "Not Found" msgstr "" -#: fff_plugin.py:2132 +#: fff_plugin.py:2205 msgid "URL is not a valid story URL." msgstr "" -#: fff_plugin.py:2135 +#: fff_plugin.py:2208 msgid "Bad URL" msgstr "" -#: fff_plugin.py:2274 fff_plugin.py:2277 +#: fff_plugin.py:2347 fff_plugin.py:2350 msgid "Anthology containing:" msgstr "" -#: fff_plugin.py:2275 +#: fff_plugin.py:2348 msgid "%s by %s" msgstr "" -#: fff_plugin.py:2297 +#: fff_plugin.py:2370 msgid " Anthology" msgstr "" -#: fff_plugin.py:2340 +#: fff_plugin.py:2413 msgid "(was set, removed for security)" msgstr "" @@ -1857,11 +1885,11 @@ msgstr "" msgid "Download started..." msgstr "" -#: jobs.py:200 +#: jobs.py:202 msgid "Already contains %d chapters. Reuse as is." msgstr "" -#: jobs.py:221 +#: jobs.py:223 msgid "Update %s completed, added %s chapters for %s total." msgstr "" From fe8e5a641c72f3995a2b741c00680a2e6c8168dc Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 17 Jul 2015 10:58:30 -0500 Subject: [PATCH 27/32] Fixes for spikeluver.com --- fanficfare/adapters/adapter_spikeluvercom.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fanficfare/adapters/adapter_spikeluvercom.py b/fanficfare/adapters/adapter_spikeluvercom.py index 937789b..bee71d3 100644 --- a/fanficfare/adapters/adapter_spikeluvercom.py +++ b/fanficfare/adapters/adapter_spikeluvercom.py @@ -106,7 +106,9 @@ class SpikeluverComAdapter(BaseSiteAdapter): listbox_tag = soup.find('div', {'class': 'listbox'}) for span_tag in listbox_tag('span'): - key = span_tag.string.strip(' :') + key = span_tag.string + if key: + key = key.strip(' :') try: value = stripHTML(span_tag.nextSibling) # This can happen with some fancy markup in the summary. Just @@ -135,8 +137,10 @@ class SpikeluverComAdapter(BaseSiteAdapter): contents.append(sibling) # Remove the preceding break line tag and other crud - contents.pop() - contents.pop() + if contents: + contents.pop() + if contents: + contents.pop() self.story.setMetadata('description', ''.join(contents)) elif key == 'Rated': From 5849f7cbbf05f8aa1f4f4ca37b7cbeb44fed6031 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 17 Jul 2015 12:15:33 -0500 Subject: [PATCH 28/32] Fix base_xenforoforum_adapter for redirected URLs with #fragments--happens if thread title changed. --- fanficfare/adapters/base_xenforoforum_adapter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fanficfare/adapters/base_xenforoforum_adapter.py b/fanficfare/adapters/base_xenforoforum_adapter.py index 4429080..f128e1a 100644 --- a/fanficfare/adapters/base_xenforoforum_adapter.py +++ b/fanficfare/adapters/base_xenforoforum_adapter.py @@ -204,8 +204,11 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): def getChapterText(self, url): logger.debug('Getting chapter text from: %s' % url) + origurl = url (data,opened) = self._fetchUrlOpened(url) url = opened.geturl() + if '#' in origurl and '#' not in url: + url = url + origurl[origurl.index('#'):] logger.debug("chapter URL redirected to: %s"%url) soup = self.make_soup(data) From ee0817fabac1437440a406dc457c3adbf3e45102 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 17 Jul 2015 13:11:38 -0500 Subject: [PATCH 29/32] Add chapter limits with URL (like [3-4]) for web service. --- webservice/ffstorage.py | 2 ++ webservice/main.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/webservice/ffstorage.py b/webservice/ffstorage.py index bad9b4a..d41a245 100644 --- a/webservice/ffstorage.py +++ b/webservice/ffstorage.py @@ -41,6 +41,8 @@ class DownloadMeta(db.Model): completed = db.BooleanProperty(default=False) date = db.DateTimeProperty(auto_now_add=True) version = db.StringProperty() + ch_begin = db.StringProperty() + ch_end = db.StringProperty() # data_chunks is implicit from DownloadData def. class DownloadData(db.Model): diff --git a/webservice/main.py b/webservice/main.py index 443fb3a..4435312 100644 --- a/webservice/main.py +++ b/webservice/main.py @@ -366,6 +366,16 @@ class FanfictionDownloader(UserConfigServer): self.redirect('/') return + # Allow chapter range with URL. + # test1.com?sid=5[4-6] + mc = re.match(r"^(?P.*?)(?:\[(?P\d+)?(?P[,-])?(?P\d+)?\])?$",url) + #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end'))) + url = mc.group('url') + ch_begin = mc.group('begin') + ch_end = mc.group('end') + if ch_begin and not mc.group('comma'): + ch_end = ch_begin + logging.info("Queuing Download: %s" % url) login = self.request.get('login') password = self.request.get('password') @@ -408,6 +418,8 @@ class FanfictionDownloader(UserConfigServer): download.title = story.getMetadata('title') download.author = story.getMetadata('author') download.url = story.getMetadata('storyUrl') + download.ch_begin = ch_begin + download.ch_end = ch_end download.put() taskqueue.add(url='/fdowntask', @@ -490,6 +502,7 @@ class FanfictionDownloaderTask(UserConfigServer): try: configuration = self.getUserConfig(user,url,format) adapter = adapters.getAdapter(configuration,url) + adapter.setChaptersRange(download.ch_begin,download.ch_end) logging.info('Created an adapter: %s' % adapter) From 61accdff32cd84901b4633b448fc41a1801832b9 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 17 Jul 2015 13:26:37 -0500 Subject: [PATCH 30/32] Add chapter limits with URL (like [3-4]) for web service. --- webservice/app.yaml | 4 ++-- webservice/index.html | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/webservice/app.yaml b/webservice/app.yaml index 9b36cec..e22aefb 100644 --- a/webservice/app.yaml +++ b/webservice/app.yaml @@ -1,6 +1,6 @@ -# ffd-retief-hrd fanfictiondownloader fanficfare +# ffd-retief-hrd fanficfare application: fanficfare -version: 2-2-10 +version: 2-2-10a runtime: python27 api_version: 1 threadsafe: true diff --git a/webservice/index.html b/webservice/index.html index a652b17..b03d5c4 100644 --- a/webservice/index.html +++ b/webservice/index.html @@ -46,7 +46,15 @@

Changes:

    -
  • Updates for mediaminer.org changes.
    • +
  • Add chapter limits with URL by giving chapter range.
    + Examples:
    +
      +
    • https://www.fanfiction.net/s/2565609/1/[4] Chapter 4 only
      • +
    • https://www.fanfiction.net/s/2565609/1/[6-10] Chapters 6, 7, 8, 9 & 10 only
      • +
    • https://www.fanfiction.net/s/2565609/1/[-10] Chapters 1-10 only
      • +
    • https://www.fanfiction.net/s/2565609/1/[150-] Chapters 150 and up only
      • +
    +

Questions? Check out our From 0e59651635783531259a92ba6e39a5b4c129cbdb Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 21 Jul 2015 10:51:16 -0500 Subject: [PATCH 31/32] Fix issue with series excluded becoming 'None' instead of ''. --- fanficfare/story.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fanficfare/story.py b/fanficfare/story.py index 9c7edc3..f90ff87 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -445,7 +445,7 @@ class Story(Configurable): self.in_ex_cludes[ie] = set_in_ex_clude(ies) def join_list(self, key, vallist): - return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist)) + return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, [ x for x in vallist if x is not None ])) def setMetadata(self, key, value, condremoveentities=True): From f72ac3c9792e6956f015710988f098d32cf6c48b Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 22 Jul 2015 08:42:47 -0500 Subject: [PATCH 32/32] Add site ninelives.dark-solace.org as Base eFiction adapter. --- calibre-plugin/plugin-defaults.ini | 6 +++ fanficfare/adapters/__init__.py | 1 + .../adapter_ninelivesdarksolaceorg.py | 38 +++++++++++++++++++ fanficfare/defaults.ini | 6 +++ 4 files changed, 51 insertions(+) create mode 100644 fanficfare/adapters/adapter_ninelivesdarksolaceorg.py diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index 7252f85..1a16d31 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -1260,6 +1260,12 @@ extracategories:NCIS extracategories:Buffy: The Vampire Slayer extracharacters:Willow +[ninelives.dark-solace.org] +## Site dedicated to these categories/characters/ships +extracategories:The Walking Dead +extracharacters:Carol,Daryl +extraships:Carol/Daryl + [nocturnal-light.net] ## Extra metadata that this adapter knows about. See [dramione.org] ## for examples of how to use them. diff --git a/fanficfare/adapters/__init__.py b/fanficfare/adapters/__init__.py index 15110b6..3865a97 100644 --- a/fanficfare/adapters/__init__.py +++ b/fanficfare/adapters/__init__.py @@ -137,6 +137,7 @@ import adapter_tgstorytimecom import adapter_itcouldhappennet import adapter_forumsspacebattlescom import adapter_forumssufficientvelocitycom +import adapter_ninelivesdarksolaceorg ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff --git a/fanficfare/adapters/adapter_ninelivesdarksolaceorg.py b/fanficfare/adapters/adapter_ninelivesdarksolaceorg.py new file mode 100644 index 0000000..22f9e78 --- /dev/null +++ b/fanficfare/adapters/adapter_ninelivesdarksolaceorg.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Fanficdownloader team, 2015 FanFicFare team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import re +from base_efiction_adapter import BaseEfictionAdapter + +class NineLivesDarkSolaceAdapter(BaseEfictionAdapter): + + @staticmethod + def getSiteDomain(): + return 'ninelives.dark-solace.org' + + @classmethod + def getSiteAbbrev(self): + return '9lvs' + + @classmethod + def getDateFormat(self): + return "%B %d, %Y" + +def getClass(): + return NineLivesDarkSolaceAdapter + diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 9f66a25..e6043bb 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -1246,6 +1246,12 @@ extracategories:NCIS extracategories:Buffy: The Vampire Slayer extracharacters:Willow +[ninelives.dark-solace.org] +## Site dedicated to these categories/characters/ships +extracategories:The Walking Dead +extracharacters:Carol,Daryl +extraships:Carol/Daryl + [nocturnal-light.net] ## Extra metadata that this adapter knows about. See [dramione.org] ## for examples of how to use them.