diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index a15c409..ade454c 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -405,7 +405,7 @@ add_to_replace_metadata: # remove [] or () blocks and leading/trailing spaces/dashes/colons title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=> # remove 'Thread' and the next word, usually "Thread 2", "Thread -# four", "Thread iv", etc +# four", "Thread iv", "Story Thread", etc title,tagsfromtitle=>[-: ]*(Story *)?[Tt]hread [^ ]+[-: ]*=> add_to_extra_titlepage_entries:,tagsfromtitle,forumtags @@ -1357,6 +1357,13 @@ extracategories:The Pretender [questionablequesting.com] ## see [base_xenforoforum] +## Some sites require login (or login for some rated stories) The +## program can prompt you, or you can save it in config. In +## commandline version, this should go in your personal.ini, not +## defaults.ini. +#username:YourName +#password:yourpassword + [samandjack.net] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 096f450..22ac269 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -338,7 +338,7 @@ class BaseSiteAdapter(Configurable): return (self._decode(data),opened) except u2.HTTPError, he: excpt=he - if he.code == 404: + if he.code in (403,404): logger.warn("Caught an exception reading URL: %s Exception %s."%(unicode(url),unicode(he))) break # break out on 404 except Exception, e: diff --git a/fanficfare/adapters/base_xenforoforum_adapter.py b/fanficfare/adapters/base_xenforoforum_adapter.py index 4084872..39b12e5 100644 --- a/fanficfare/adapters/base_xenforoforum_adapter.py +++ b/fanficfare/adapters/base_xenforoforum_adapter.py @@ -87,6 +87,42 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): ''' return True + def performLogin(self): + params = {} + + if self.password: + params['login'] = self.username + params['password'] = self.password + else: + params['login'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['register'] = '0' + params['cookie_check'] = '1' + params['_xfToken'] = '' + params['redirect'] = 'https://' + self.getSiteDomain() + '/' + + if not params['password']: + return + + ## https://forum.questionablequesting.com/login/login + loginUrl = 'https://' + self.getSiteDomain() + '/login/login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['login'])) + + # soup = self.make_soup(self._fetchUrl(loginUrl)) + # params['ctkn']=soup.find('input', {'name':'ctkn'})['value'] + # params[soup.find('input', {'id':'password'})['name']] = params['password'] + + d = self._fetchUrl(loginUrl, params) + + if "Log Out" not in d : + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['login'])) + raise exceptions.FailedToLogin(self.url,params['login']) + return False + else: + return True + ## Getting the chapter list and the meta data, plus 'is adult' checking. def extractChapterUrlsAndMetadata(self): @@ -100,8 +136,13 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): except urllib2.HTTPError, e: if e.code == 404: raise exceptions.StoryDoesNotExist(self.url) + elif e.code == 403: + self.performLogin() + (data,opened) = self._fetchUrlOpened(useurl) + useurl = opened.geturl() + logger.info("use useurl: "+useurl) else: - raise e + raise # use BeautifulSoup HTML parser to make everything easier to find. soup = self.make_soup(data) diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 357cb83..e1bb28e 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -404,7 +404,7 @@ add_to_replace_metadata: # remove [] or () blocks and leading/trailing spaces/dashes/colons title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=> # remove 'Thread' and the next word, usually "Thread 2", "Thread -# four", "Thread iv", etc +# four", "Thread iv", "Story Thread", etc title,tagsfromtitle=>[-: ]*(Story *)?[Tt]hread [^ ]+[-: ]*=> add_to_extra_titlepage_entries:,tagsfromtitle,forumtags @@ -1345,6 +1345,13 @@ extracategories:The Pretender [questionablequesting.com] ## see [base_xenforoforum] +## Some sites require login (or login for some rated stories) The +## program can prompt you, or you can save it in config. In +## commandline version, this should go in your personal.ini, not +## defaults.ini. +#username:YourName +#password:yourpassword + [samandjack.net] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In