From 572f4e4c7fbb192d8416833252d0449391f8d858 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 26 Feb 2016 10:30:02 -0600 Subject: [PATCH] Adding [storyUrl] ini sections feature for custom config for individual stories. --- calibre-plugin/fff_plugin.py | 20 ++++++- calibre-plugin/inihighlighter.py | 10 +++- fanficfare/adapters/__init__.py | 5 +- fanficfare/adapters/adapter_test1.py | 1 - fanficfare/adapters/base_adapter.py | 3 + fanficfare/configurable.py | 30 ++++++++-- fanficfare/geturls.py | 6 +- fanficfare/story.py | 85 ++++++++++++++++++---------- 8 files changed, 115 insertions(+), 45 deletions(-) diff --git a/calibre-plugin/fff_plugin.py b/calibre-plugin/fff_plugin.py index cb60412..ea07f9b 100644 --- a/calibre-plugin/fff_plugin.py +++ b/calibre-plugin/fff_plugin.py @@ -7,6 +7,22 @@ __license__ = 'GPL v3' __copyright__ = '2016, Jim Miller' __docformat__ = 'restructuredtext en' + +# import cProfile + +# def do_cprofile(func): +# def profiled_func(*args, **kwargs): +# profile = cProfile.Profile() +# try: +# profile.enable() +# result = func(*args, **kwargs) +# profile.disable() +# return result +# finally: +# profile.print_stats() +# return profiled_func + + import logging logger = logging.getLogger(__name__) @@ -994,7 +1010,8 @@ class FanFicFarePlugin(InterfaceAction): show_copy_button=False): rejecturllist.remove(url) return False - + + # @do_cprofile def prep_download_loop(self,book, options={'fileform':'epub', 'collision':ADDNEW, @@ -1015,6 +1032,7 @@ class FanFicFarePlugin(InterfaceAction): ## Check reject list. Redundant with below for when story URL ## changes, but also kept here to avoid network hit in most ## common case where given url is story url. + if self.reject_url(merge,book): return diff --git a/calibre-plugin/inihighlighter.py b/calibre-plugin/inihighlighter.py index 7348d80..131ca43 100644 --- a/calibre-plugin/inihighlighter.py +++ b/calibre-plugin/inihighlighter.py @@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, print_function) __license__ = 'GPL v3' -__copyright__ = '2015, Jim Miller' +__copyright__ = '2016, Jim Miller' __docformat__ = 'restructuredtext en' import re @@ -68,6 +68,10 @@ class IniHighlighter(QSyntaxHighlighter): self.teststoryRule = HighlightingRule( r"^\[teststory:([0-9]+|defaults)\]", Qt.darkCyan, blocknum=3 ) self.highlightingRules.append( self.teststoryRule ) + # storyUrl sections + self.storyUrlRule = HighlightingRule( r"^\[https?://.*\]", Qt.darkMagenta, blocknum=4 ) + self.highlightingRules.append( self.storyUrlRule ) + # NOT comments -- but can be custom columns, so don't flag. #self.highlightingRules.append( HighlightingRule( r"(? Some more longer description. "I suck at summaries!" "Better than it sounds!" "My first fic" diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index a1dc29e..70dc1d5 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -188,6 +188,9 @@ class BaseSiteAdapter(Configurable): self.host = self.parsedUrl.netloc self.path = self.parsedUrl.path self.story.setMetadata('storyUrl',self.url,condremoveentities=False) + self.addUrlConfigSection(url) # self.story shares the same configuration. + # ignored inside if config is_lightweight() + self.story.config_prepped = False ## website encoding(s)--in theory, each website reports the character ## encoding they use for each page. In practice, some sites report it diff --git a/fanficfare/configurable.py b/fanficfare/configurable.py index c830b60..ca25375 100644 --- a/fanficfare/configurable.py +++ b/fanficfare/configurable.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015 Fanficdownloader team, 2015 FanFicFare team +# Copyright 2015 Fanficdownloader team, 2016 FanFicFare team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -365,10 +365,12 @@ def make_generate_cover_settings(param): class Configuration(ConfigParser.SafeConfigParser): - def __init__(self, sections, fileform): + def __init__(self, sections, fileform, lightweight=False): site = sections[-1] # first section is site DN. ConfigParser.SafeConfigParser.__init__(self) + self.lightweight = lightweight + self.linenos=dict() # key by section or section,key -> lineno ## [injected] section has even less priority than [defaults] @@ -403,8 +405,18 @@ class Configuration(ConfigParser.SafeConfigParser): self.validEntries = get_valid_entries() - def addConfigSection(self,section): - self.sectionslist.insert(0,section) + def addUrlConfigSection(self,url): + if not self.lightweight: # don't need when just checking for normalized URL. + self.addConfigSection(url,'overrides') + + def addConfigSection(self,section,before=None): + if section not in self.sectionslist: # don't add if already present. + if before is None: + self.sectionslist.insert(0,section) + else: + ## because sectionslist is hi-pri first, lo-pri last, + ## 'before' means after in the list. + self.sectionslist.insert(self.sectionslist.index(before)+1,section) def isListType(self,key): return key in self.listTypeEntries or self.hasConfig("include_in_"+key) @@ -604,7 +616,7 @@ class Configuration(ConfigParser.SafeConfigParser): def test_config(self): errors=[] - teststory_re = re.compile(r'^teststory:(defaults|[0-9]+)$') + allowedsections_re = re.compile(r'^(teststory:(defaults|[0-9]+)|https?://.*)$') allowedsections = get_valid_sections() clude_metadata_re = re.compile(r'(add_to_)?(in|ex)clude_metadata_(pre|post)') @@ -619,7 +631,7 @@ class Configuration(ConfigParser.SafeConfigParser): valdict = get_valid_set_options() for section in self.sections(): - if section not in allowedsections and not teststory_re.match(section): + if section not in allowedsections and not allowedsections_re.match(section): errors.append((self.get_lineno(section),"Bad Section Name: [%s]"%section)) else: sitename = section.replace('www.','') @@ -685,6 +697,12 @@ class Configurable(object): def __init__(self, configuration): self.configuration = configuration + def is_lightweight(self): + return self.configuration.lightweight + + def addUrlConfigSection(self,url): + self.configuration.addUrlConfigSection(url) + def isListType(self,key): return self.configuration.isListType(key) diff --git a/fanficfare/geturls.py b/fanficfare/geturls.py index 9b63c39..5da585c 100644 --- a/fanficfare/geturls.py +++ b/fanficfare/geturls.py @@ -35,7 +35,7 @@ from exceptions import UnknownSite def get_urls_from_page(url,configuration=None,normalize=False): if not configuration: - configuration = Configuration("test1.com","EPUB") + configuration = Configuration(["test1.com"],"EPUB",lightweight=True) data = None adapter = None @@ -84,7 +84,7 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict urls = collections.OrderedDict() if not configuration: - configuration = Configuration("test1.com","EPUB") + configuration = Configuration(["test1.com"],"EPUB",lightweight=True) soup = BeautifulSoup(data,"html5lib") if restrictsearch: @@ -128,7 +128,7 @@ def get_urls_from_text(data,configuration=None,normalize=False): data=unicode(data) if not configuration: - configuration = Configuration("test1.com","EPUB") + configuration = Configuration(["test1.com"],"EPUB",lightweight=True) for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data): # this (should) catch normal story links, some javascript diff --git a/fanficfare/story.py b/fanficfare/story.py index b8488f7..a639bfb 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2011 Fanficdownloader team, 2015 FanFicFare team +# Copyright 2011 Fanficdownloader team, 2016 FanFicFare team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -380,8 +380,8 @@ def set_in_ex_clude(setting): ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp] def make_replacements(replace): retval=[] - for fullline in replace.splitlines(): - line=fullline + for repl_line in replace.splitlines(): + line=repl_line try: (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None) if "&&" in line: @@ -403,10 +403,10 @@ def make_replacements(replace): # replacement string. The .ini parser eats any # trailing spaces. replacement=replacement.replace(SPACE_REPLACE,' ') - retval.append([metakeys,regexp,replacement,condkey,condregexp]) + retval.append([repl_line,metakeys,regexp,replacement,condkey,condregexp]) except Exception as e: - logger.error("Problem with Replacement Line:%s"%fullline) - raise exceptions.PersonalIniFailed(e,'replace_metadata unpacking failed',fullline) + logger.error("Problem with Replacement Line:%s"%repl_line) + raise exceptions.PersonalIniFailed(e,'replace_metadata unpacking failed',repl_line) # raise return retval @@ -419,6 +419,7 @@ class Story(Configurable): self.metadata = {'version':os.environ['CURRENT_VERSION_ID']} except: self.metadata = {'version':'4.4'} + self.replacements = [] self.in_ex_cludes = {} self.chapters = [] # chapters will be namedtuple of Chapter(url,title,html,etc) self.chapter_first = None @@ -431,26 +432,35 @@ class Story(Configurable): self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update. self.logfile=None # cheesy way to carry log file forward across update. - ## Look for config parameter, split and add each to metadata field. - for (config,metadata) in [("extracategories","category"), - ("extragenres","genre"), - ("extracharacters","characters"), - ("extraships","ships"), - ("extrawarnings","warnings")]: - for val in self.getConfigList(config): - self.addToList(metadata,val) + self.replacements_prepped = False - self.replacements = make_replacements(self.getConfig('replace_metadata')) - - in_ex_clude_list = ['include_metadata_pre','exclude_metadata_pre', - 'include_metadata_post','exclude_metadata_post'] - for ie in in_ex_clude_list: - ies = self.getConfig(ie) - # print("%s %s"%(ie,ies)) - if ies: - iel = [] - self.in_ex_cludes[ie] = set_in_ex_clude(ies) + def prepare_replacements(self): + if not self.replacements_prepped and not self.is_lightweight(): + logger.debug("prepare_replacements") + logger.debug("sections:%s"%self.configuration.sectionslist) + + ## Look for config parameter, split and add each to metadata field. + for (config,metadata) in [("extracategories","category"), + ("extragenres","genre"), + ("extracharacters","characters"), + ("extraships","ships"), + ("extrawarnings","warnings")]: + for val in self.getConfigList(config): + self.addToList(metadata,val) + + self.replacements = make_replacements(self.getConfig('replace_metadata')) + + in_ex_clude_list = ['include_metadata_pre','exclude_metadata_pre', + 'include_metadata_post','exclude_metadata_post'] + for ie in in_ex_clude_list: + ies = self.getConfig(ie) + # print("%s %s"%(ie,ies)) + if ies: + iel = [] + self.in_ex_cludes[ie] = set_in_ex_clude(ies) + self.replacements_prepped = True + def set_chapters_range(self,first=None,last=None): self.chapter_first=first self.chapter_last=last @@ -484,6 +494,10 @@ class Story(Configurable): def do_in_ex_clude(self,which,value,key): + # sets self.replacements and self.in_ex_cludes if needed + # do_in_ex_clude is always called from doReplacements, so redundant. + # self.prepare_replacements() + if value and which in self.in_ex_cludes: include = 'include' in which keyfound = False @@ -512,8 +526,10 @@ class Story(Configurable): value = None return value - def doReplacements(self,value,key,return_list=False,seen_list=[]): + # sets self.replacements and self.in_ex_cludes if needed + self.prepare_replacements() + value = self.do_in_ex_clude('include_metadata_pre',value,key) value = self.do_in_ex_clude('exclude_metadata_pre',value,key) @@ -523,7 +539,7 @@ class Story(Configurable): # print("bailing on %s"%replaceline) continue #print("replacement tuple:%s"%replaceline) - (metakeys,regexp,replacement,condkey,condregexp) = replaceline + (repl_line,metakeys,regexp,replacement,condkey,condregexp) = replaceline if (metakeys == None or key in metakeys) \ and isinstance(value,basestring) \ and regexp.search(value): @@ -541,21 +557,28 @@ class Story(Configurable): if SPLIT_META in replacement: retlist = [] for splitrepl in replacement.split(SPLIT_META): - retlist.extend(self.doReplacements(regexp.sub(splitrepl,value), + try: + tval = regexp.sub(splitrepl,value) + except: + logger.error("Exception with replacement line,value:(%s),(%s)"%(repl_line,value)) + raise + retlist.extend(self.doReplacements(tval, key, return_list=True, seen_list=seen_list+[replaceline])) break else: # print("replacement,value:%s,%s->%s"%(replacement,value,regexp.sub(replacement,value))) - value = regexp.sub(replacement,value) - retlist = [value] + try: + value = regexp.sub(replacement,value) + retlist = [value] + except: + logger.error("Exception with replacement line,value:(%s),(%s)"%(repl_line,value)) + raise for val in retlist: retlist = map(partial(self.do_in_ex_clude,'include_metadata_post',key=key),retlist) retlist = map(partial(self.do_in_ex_clude,'exclude_metadata_post',key=key),retlist) - # value = self.do_in_ex_clude('include_metadata_post',value,key) - # value = self.do_in_ex_clude('exclude_metadata_post',value,key) if return_list: return retlist