From 1ae2c396476f51e66cd572deeb46dbd379190986 Mon Sep 17 00:00:00 2001 From: Ida Date: Fri, 8 Jun 2012 10:39:52 -0400 Subject: [PATCH] Added login part for onedirectionfanfiction --- .../adapter_onedirectionfanfictioncom.py | 65 ++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py index 3d9afff..27c87a6 100644 --- a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py @@ -68,23 +68,68 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter): def getSiteDomain(): # The site domain. Does have www here, if it uses it. return 'onedirectionfanfiction.com' - + @classmethod def getAcceptDomains(cls): return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com'] + def getSiteExampleURLs(self): return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234" def getSiteURLPattern(self): return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logging.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True ## Getting the chapter list and the meta data, plus 'is adult' checking. def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" # index=1 makes sure we see the story chapter index. Some # sites skip that for one-chapter stories. - url = self.url+'&index=1' + url = self.url+'&index=1'+addurl logging.debug("URL: "+url) try: @@ -98,6 +143,20 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter): # use BeautifulSoup HTML parser to make everything easier to find. soup = bs.BeautifulSoup(data) # print data + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if "Age Consent Required" in data: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") # Now go hunting for all the meta data and the chapter list. @@ -114,7 +173,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter): # Find the chapters: for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) self.story.setMetadata('numChapters',len(self.chapterUrls))