diff --git a/fanficfare/adapters/__init__.py b/fanficfare/adapters/__init__.py index 37c3b9b..4717ac6 100644 --- a/fanficfare/adapters/__init__.py +++ b/fanficfare/adapters/__init__.py @@ -85,7 +85,6 @@ import adapter_hpfanficarchivecom import adapter_twilightarchivescom import adapter_nhamagicalworldsus import adapter_hlfictionnet -import adapter_grangerenchantedcom import adapter_dracoandginnycom import adapter_scarvesandcoffeenet import adapter_thepetulantpoetesscom @@ -102,13 +101,9 @@ import adapter_efictionestelielde import adapter_pommedesangcom import adapter_restrictedsectionorg import adapter_imagineeficcom -import adapter_buffynfaithnet import adapter_psychficcom -import adapter_tokrafandomnetcom import adapter_asr3slashzoneorg -import adapter_nickandgregnet import adapter_potterheadsanonymouscom -import adapter_scarheadnet import adapter_fictionpadcom import adapter_storiesonlinenet import adapter_trekiverseorg @@ -120,7 +115,6 @@ import adapter_nocturnallightnet import adapter_fanfichu import adapter_fanfictioncsodaidokhu import adapter_fictionmaniatv -import adapter_bdsmgeschichten import adapter_tolkienfanfiction import adapter_themaplebookshelf import adapter_fannation @@ -139,13 +133,10 @@ import adapter_ninelivesarchivecom import adapter_masseffect2in import adapter_quotevcom import adapter_mcstoriescom - import adapter_lucifaelff import adapter_buffygilescom -#import adapter_rubyquillcom -import adapter_andromedawebcom # Not all lables are captured +import adapter_andromedawebcom import adapter_artemisfowlcom -import adapter_rabidreadercom import adapter_naiceanilmenet ## This bit of complexity allows adapters to be added by just adding diff --git a/fanficfare/adapters/adapter_bdsmgeschichten.py b/fanficfare/adapters/adapter_bdsmgeschichten.py deleted file mode 100644 index 70b10e5..0000000 --- a/fanficfare/adapters/adapter_bdsmgeschichten.py +++ /dev/null @@ -1,347 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2014 Fanficdownloader team, 2015 FanFicFare team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time -import logging -logger = logging.getLogger(__name__) -import re -import urllib2 -import urlparse -import time - -from bs4.element import Tag, Comment -from .. import exceptions as exceptions - -from base_adapter import BaseSiteAdapter, makeDate - -def _translate_date_german_english(date): - fullmon = {"Januar":"01", - "Februar":"02", - u"März":"03", - "April":"04", - "Mai":"05", - "Juni":"06", - "Juli":"07", - "August":"08", - "September":"09", - "Oktober":"10", - "November":"11", - "Dezember":"12"} - for (name,num) in fullmon.items(): - date = date.replace(name,num) - return date - -_REGEX_TRAILING_DIGIT = re.compile("(\d+)$") -_REGEX_DASH_TO_END = re.compile("-[^-]+$") -_REGEX_CHAPTER_TITLE = re.compile(ur""" - \s* - [\u2013-]? - \s* - ([\dIVX-]+)? - \.? - \s* - [\[\(]? - \s* - (Teil|Kapitel|Tag)? - \s* - ([\dIVX-]+)? - \s* - [\]\)]? - \s* - $ -""", re.VERBOSE) -_INITIAL_STEP = 5 - -class BdsmGeschichtenAdapter(BaseSiteAdapter): - - def __init__(self, config, url): - BaseSiteAdapter.__init__(self, config, url) - - self.decode = ["utf8", "Windows-1252"] - - self.story.setMetadata('siteabbrev','bdsmgesch') - - # Replace possible chapter numbering - chapterMatch = _REGEX_TRAILING_DIGIT.search(url) - if chapterMatch is None: - self.maxChapter = 1 - else: - self.maxChapter = int(chapterMatch.group(1)) - # url = re.sub(_REGEX_TRAILING_DIGIT, "1", url) - - # set storyId - self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId')) - - # normalize URL - self._setURL('http://%s/%s' % (self.getSiteDomain(), self.story.getMetadata('storyId'))) - - self.dateformat = '%d. %m %Y - %H:%M' - - @staticmethod - def getSiteDomain(): - return 'bdsm-geschichten.net' - - @classmethod - def getAcceptDomains(cls): - return ['www.bdsm-geschichten.net', 'www.bdsm-geschichten.net'] - - @classmethod - def getSiteExampleURLs(cls): - return "http://www.bdsm-geschichten.net/title-of-story-1 http://bdsm-geschichten.net/title-of-story-1" - - def getSiteURLPattern(self): - return r"http://(www\.)?bdsm-geschichten.net/(?P[a-zA-Z0-9_-]+)" - - def extractChapterUrlsAndMetadata(self): - - if not (self.is_adult or self.getConfig("is_adult")): - raise exceptions.AdultCheckRequired(self.url) - - try: - data1 = self._fetchUrl(self.url) - soup = self.make_soup(data1) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - - #strip comments from soup - [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))] - - # Cache the soups so we won't have to redownload in getChapterText later - self.soupsCache = {} - self.soupsCache[self.url] = soup - - # author - authorDiv = soup.find("div", "author-pane-line author-name") - authorId = authorDiv.string.strip() - self.story.setMetadata('authorId', authorId) - self.story.setMetadata('author', authorId) - # TODO not really true need to be loggedin for this to work or fetch userid - self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId) - - # TODO better metadata - date = soup.find("div", {"class": "submitted"}).string.strip() - # 11. April 2015 - 17:08 - date = re.sub(r"(\d+\. \D+ \d+ - \d+:\d+).*", r"\1", date) - date = _translate_date_german_english(date) - self.story.setMetadata('datePublished', makeDate(date, self.dateformat)) - title1 = soup.find("h1", {'class': 'title'}).string - - - for tagLink in soup.find("ul", "taxonomy").findAll("a"): - self.story.addToList('category', tagLink.string) - - ## Retrieve chapter soups - if self.getConfig('find_chapters') == 'guess': - self.chapterUrls = [] - self._find_chapters_by_guessing(title1) - else: - self._find_chapters_by_parsing(soup) - - firstChapterUrl = self.chapterUrls[0][1] - if firstChapterUrl in self.soupsCache: - firstChapterSoup = self.soupsCache[firstChapterUrl] - h1 = firstChapterSoup.find("h1").text - else: - h1 = soup.find("h1").text - - h1 = re.sub(_REGEX_CHAPTER_TITLE, "", h1) - self.story.setMetadata('title', h1) - self.story.setMetadata('numChapters', len(self.chapterUrls)) - return - - def _find_chapters_by_parsing(self, soup): - - # store original soup - origSoup = soup - - # - # find first chapter - # - firstLink = None - firstLinkDiv = soup.find("div", "field-field-erster-teil") - if firstLinkDiv is not None: - firstLink = "http://%s%s" % (self.getSiteDomain(), firstLinkDiv.findNext("a")['href']) - logger.debug("Found first chapter right away <%s>" % firstLink) - try: - soup = self.make_soup(self._fetchUrl(firstLink)) - self.soupsCache[firstLink] = soup - self.chapterUrls.insert(0, (soup.find("h1").text, firstLink)) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise exceptions.StoryDoesNotExist(firstLink) - else: - logger.debug("DIDN'T find first chapter right away") - # parse previous Link until first - while True: - prevLink = None - prevLinkDiv = soup.find("div", "field-field-vorheriger-teil") - if prevLinkDiv is not None: - prevLink = prevLinkDiv.find("a") - if prevLink is None: - prevLink = soup.find("a", text=re.compile("<<<")) # <<< - if prevLink is None: - logger.debug("Couldn't find prev part") - break - else: - logger.debug("Previous Chapter <%s>" % prevLink) - if type(prevLink) != Tag or prevLink.name != "a": - prevLink = prevLink.findParent("a") - if prevLink is None or '#' in prevLink['href']: - logger.debug("Couldn't find prev part (false positive) <%s>" % prevLink) - break - prevLink = prevLink['href'] - try: - soup = self.make_soup(self._fetchUrl(prevLink)) - self.soupsCache[prevLink] = soup - prevTtitle = soup.find("h1", {'class': 'title'}).string - self.chapterUrls.insert(0, (prevTtitle, prevLink)) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(nextLink) - else: - raise e - firstLink = prevLink - - # if first chapter couldn't be determined, assume the URL originally - # passed is the first chapter - if firstLink is None: - logger.debug("Couldn't set first chapter") - firstLink = self.url - self.chapterUrls.insert(0, (soup.find("h1").text, firstLink)) - - # set first URL - logger.debug("Set first link: %s" % firstLink) - self._setURL(firstLink) - self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(firstLink).group('storyId')) - - # - # Parse next chapters - # - while True: - nextLink = None - nextLinkDiv = soup.find("div", "field-field-naechster-teil") - if nextLinkDiv is not None: - nextLink = nextLinkDiv.find("a") - if nextLink is None: - nextLink = soup.find("a", text=re.compile(">>>")) - if nextLink is None: - nextLink = soup.find("a", text=re.compile("Fortsetzung")) - - if nextLink is None: - logger.debug("Couldn't find next part") - break - else: - if type(nextLink) != Tag or nextLink.name != "a": - nextLink = nextLink.findParent("a") - if nextLink is None or '#' in nextLink['href']: - logger.debug("Couldn't find next part (false positive) <%s>" % nextLink) - break - nextLink = nextLink['href'] - - if not nextLink.startswith('http:'): - nextLink = 'http://' + self.getSiteDomain() + nextLink - - for loadedChapter in self.chapterUrls: - if loadedChapter[0] == nextLink: - logger.debug("ERROR: Repeating chapter <%s> Try to fix it" % nextLink) - nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink) - if nextLinkMatch is not None: - curChap = nextLinkMatch.group(1) - nextLink = re.sub(_REGEX_TRAILING_DIGIT, unicode(int(curChap) + 1), nextLink) - else: - break - try: - data = self._fetchUrl(nextLink) - soup = self.make_soup(data) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(nextLink) - else: - raise e - title2 = soup.find("h1", {'class': 'title'}).string - self.chapterUrls.append((title2, nextLink)) - logger.debug("Grabbing next chapter URL " + nextLink) - self.soupsCache[nextLink] = soup - # [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))] - logger.debug("Chapters: %s" % self.chapterUrls) - - - def _find_chapters_by_guessing(self, title1): - step = _INITIAL_STEP - curMax = self.maxChapter + step - lastHit = True - while True: - nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, unicode(curMax), self.url) - if nextChapterUrl == self.url: - logger.debug("Unable to guess next chapter because URL doesn't end in numbers") - break; - try: - logger.debug("Trying chapter URL " + nextChapterUrl) - data = self._fetchUrl(nextChapterUrl) - hit = True - except urllib2.HTTPError, e: - if e.code == 404: - hit = False - else: - raise e - if hit: - logger.debug("Found chapter URL " + nextChapterUrl) - self.maxChapter = curMax - self.soupsCache[nextChapterUrl] = self.make_soup(data) - if not lastHit: - break - lastHit = curMax - curMax += step - else: - lastHit = False - curMax -= 1 - logger.debug(curMax) - - for i in xrange(1, self.maxChapter): - nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, unicode(i), self.url) - nextChapterTitle = re.sub("1", unicode(i), title1) - self.chapterUrls.append((nextChapterTitle, nextChapterUrl)) - - def getChapterText(self, url): - - if url in self.soupsCache: - logger.debug('Getting chapter <%s> from cache' % url) - soup = self.soupsCache[url] - else: - logger.debug('Downloading chapter <%s>' % url) - data1 = self._fetchUrl(url) - soup = self.make_soup(data1) - #strip comments from soup - [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))] - - # get story text - storyDiv1 = soup.new_tag("div") - for para in soup.find("div", "full-node").find('div', 'content').findAll("p"): - storyDiv1.append(para) - storyDiv1.append(soup.new_tag("br")) - storytext = self.utf8FromSoup(url,storyDiv1) - - return storytext - - -def getClass(): - return BdsmGeschichtenAdapter diff --git a/fanficfare/adapters/adapter_buffynfaithnet.py b/fanficfare/adapters/adapter_buffynfaithnet.py deleted file mode 100644 index c7858a4..0000000 --- a/fanficfare/adapters/adapter_buffynfaithnet.py +++ /dev/null @@ -1,292 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time -import logging -logger = logging.getLogger(__name__) -import re -import urllib2 -import cookielib as cl - - -from ..htmlcleanup import stripHTML -from .. import exceptions as exceptions - -from base_adapter import BaseSiteAdapter, makeDate - - -# This function is called by the downloader in all adapter_*.py files -# in this dir to register the adapter class. So it needs to be -# updated to reflect the class below it. That, plus getSiteDomain() -# take care of 'Registering'. -def getClass(): - return BuffyNFaithNetAdapter - -# Class name has to be unique. Our convention is camel case the -# sitename with Adapter at the end. www is skipped. -class BuffyNFaithNetAdapter(BaseSiteAdapter): - - def __init__(self, config, url): - BaseSiteAdapter.__init__(self, config, url) - - self.setHeader() - - self.decode = ["Windows-1252", - "utf8"] # 1252 is a superset of iso-8859-1. - # Most sites that claim to be - # iso-8859-1 (and some that claim to be - # utf8) are really windows-1252. - self.username = "NoneGiven" # if left empty, site doesn't return any message at all. - self.password = "" - self.is_adult=False - - # get storyId from url--url validation guarantees query correct - m = re.match(self.getSiteURLPattern(),url) - if m: - self.story.setMetadata('storyId',m.group('id')) - - # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site - nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId') - self._setURL(nurl) - else: - raise exceptions.InvalidStoryURL(url, - self.getSiteDomain(), - self.getSiteExampleURLs()) - - - # Each adapter needs to have a unique site abbreviation. - self.story.setMetadata('siteabbrev','bnfnet') - - @staticmethod # must be @staticmethod, don't remove it. - def getSiteDomain(): - # The site domain. Does have www here, if it uses it. - return 'buffynfaith.net' - - @classmethod - def stripURLParameters(cls,url): - "Only needs to be overriden if URL contains more than one parameter" - ## This adapter needs at least two parameters left on the URL, act and id - return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url) - - def setHeader(self): - "buffynfaith.net wants a Referer for images. Used both above and below(after cookieproc added)" - self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/')) - - @classmethod - def getSiteExampleURLs(cls): - return "http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=ovr&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234&ch=2" - - def getSiteURLPattern(self): - #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963 - #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949 - #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2 - p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\ - r"(vie|ovr)&id=(?P\d+)(&ch=(?P\d+))?$" - return p - - def use_pagecache(self): - ''' - adapters that will work with the page cache need to implement - this and change it to True. - ''' - return True - - def extractChapterUrlsAndMetadata(self): - - dateformat = "%d %B %Y" - url = self.url - logger.debug("URL: "+url) - - #set a cookie to get past adult check - if self.is_adult or self.getConfig("is_adult"): - cookie = cl.Cookie(version=0, name='my_age', value='yes', - port=None, port_specified=False, - domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False, - path='/', path_specified=True, - secure=False, - expires=time.time()+10000, - discard=False, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False) - self.get_cookiejar().set_cookie(cookie) - self.setHeader() - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - - #print data - - if "ADULT CONTENT WARNING" in data: - raise exceptions.AdultCheckRequired(self.url) - - # use BeautifulSoup HTML parser to make everything easier to find. - soup = self.make_soup(data) - - # Now go hunting for all the meta data and the chapter list. - - #stuff in : description - svalue = soup.head.find('meta',attrs={'name':'description'})['content'] - #self.story.setMetadata('description',svalue) - self.setDescription(url,svalue) - - #useful stuff in rest of doc, all contained in this: - doc = soup.body.find('div', id='my_wrapper') - - #first the site category (more of a genre to me, meh) and title, in this element: - mt = doc.find('div',attrs={'class':'maintitle'}) - self.story.addToList('genre',mt.findAll('a')[1].string) - self.story.setMetadata('title',stripHTML(mt).split(u'»')[-1].strip()) - del mt - - #the actual category, for me, is 'Buffy: The Vampire Slayer' - #self.story.addToList('category','Buffy: The Vampire Slayer') - #No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini - - #then a block that sits in a table cell like so: - #(contains a lot of metadata) - mblock = doc.find('td', align='left', width = '70%').contents - while len(mblock) > 0: - i = mblock.pop(0) - if 'Author:' in i.string: - #drop empty space - mblock.pop(0) - #get author link - a = mblock.pop(0) - authre = re.escape('./index.php?act=bio&id=')+'(?P\d+)' - m = re.match(authre,a['href']) - self.story.setMetadata('author',a.string) - self.story.setMetadata('authorId',m.group('authid')) - authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(), - self.story.getMetadata('authorId')) - self.story.setMetadata('authorUrl',authurl,condremoveentities=False) - #drop empty space - mblock.pop(0) - if 'Rating:' in i.string: - self.story.setMetadata('rating',mblock.pop(0).strip()) - if 'Published:' in i.string: - date = mblock.pop(0).strip() - #get rid of 'st', 'nd', 'rd', 'th' after day number - date = date[0:2]+date[4:] - self.story.setMetadata('datePublished',makeDate(date, dateformat)) - if 'Last Updated:' in i.string: - date = mblock.pop(0).strip() - #get rid of 'st', 'nd', 'rd', 'th' after day number - date = date[0:2]+date[4:] - self.story.setMetadata('dateUpdated',makeDate(date, dateformat)) - if 'Genre:' in i.string: - genres = mblock.pop(0).strip() - genres = genres.split('/') - for genre in genres: self.story.addToList('genre',genre) - #end ifs - #end while - - # Find the chapter selector - select = soup.find('select', { 'name' : 'ch' } ) - - if select is None: - # no selector found, so it's a one-chapter story. - #self.chapterUrls.append((self.story.getMetadata('title'),url)) - self.chapterUrls.append((self.story.getMetadata('title'),url)) - else: - allOptions = select.findAll('option') - for o in allOptions: - url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(), - self.story.getMetadata('storyId'), - o['value']) - title = u"%s" % o - title = stripHTML(title) - ts = title.split(' ',1) - title = ts[0]+'. '+ts[1] - self.chapterUrls.append((title,url)) - self.story.setMetadata('numChapters',len(self.chapterUrls)) - - ## Go scrape the rest of the metadata from the author's page. - data = self._fetchUrl(self.story.getMetadata('authorUrl')) - soup = self.make_soup(data) - #find the story link and its parent div - storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')}) - storydiv = storya.parent - #warnings come under a tag. Never seen that before... - #appears to just be a line of freeform text, not necessarily a list - #optional - spawn = storydiv.find('spawn',{'id':'warnings'}) - if spawn is not None: - warns = spawn.nextSibling.strip() - self.story.addToList('warnings',warns) - #some meta in spans - this should get all, even the ones jammed in a table - spans = storydiv.findAll('span') - for s in spans: - if s.string == 'Ship:': - list = s.nextSibling.strip().split() - self.story.extendList('ships',list) - if s.string == 'Characters:': - list = s.nextSibling.strip().split(',') - self.story.extendList('characters',list) - if s.string == 'Status:': - st = s.nextSibling.strip() - self.story.setMetadata('status',st) - if s.string == 'Words:': - st = s.nextSibling.strip() - self.story.setMetadata('numWords',st) - - #reviews - is this worth having? - #ffnet adapter gathers it, don't know if anything else does - #or if it's ever going to be used! - a = storydiv.find('a',{'id':'bold-blue'}) - if a: - revs = a.nextSibling.strip()[1:-1] - self.story.setMetadata('reviews',st) - else: - revs = '0' - self.story.setMetadata('reviews',st) - - # grab the text for an individual chapter. - def getChapterText(self, url): - - logger.debug('Getting chapter text from: %s' % url) - - soup = self.make_soup(self._fetchUrl(url)) - - div = soup.find('div', {'id' : 'fanfiction'}) - if None == div: - raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - - #remove all the unnecessary bookmark tags - [s.extract() for s in div('div',{'class':"tiny_box2"})] - - #is there a review link? - r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$")) - if r is not None: - #remove the review link and its parent div - r.parent.extract() - - #There might also be a link to the sequel on the last chapter - #I'm inclined to keep it in, but the URL needs to be changed from relative to absolute - #Shame there isn't proper series metadata available - #(I couldn't find it anyway) - s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$")) - if s is not None: - s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:] - - return self.utf8FromSoup(url,div) diff --git a/fanficfare/adapters/adapter_grangerenchantedcom.py b/fanficfare/adapters/adapter_grangerenchantedcom.py deleted file mode 100644 index 35b67e3..0000000 --- a/fanficfare/adapters/adapter_grangerenchantedcom.py +++ /dev/null @@ -1,309 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2012 Fanficdownloader team, 2015 FanFicFare team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Software: eFiction -import time -import logging -logger = logging.getLogger(__name__) -import re -import urllib2 - - -from ..htmlcleanup import stripHTML -from .. import exceptions as exceptions - -from base_adapter import BaseSiteAdapter, makeDate - -def getClass(): - return GrangerEnchantedCom - -# Class name has to be unique. Our convention is camel case the -# sitename with Adapter at the end. www is skipped. -class GrangerEnchantedCom(BaseSiteAdapter): - - def __init__(self, config, url): - BaseSiteAdapter.__init__(self, config, url) - - self.decode = ["Windows-1252", - "utf8"] # 1252 is a superset of iso-8859-1. - # Most sites that claim to be - # iso-8859-1 (and some that claim to be - # utf8) are really windows-1252. - self.username = "NoneGiven" # if left empty, site doesn't return any message at all. - self.password = "" - self.is_adult=False - - # get storyId from url--url validation guarantees query is only sid=1234 - self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) - - self.section=self.parsedUrl.path.split('/',)[1] - - # normalized story URL. - if "malfoymanor" in self.parsedUrl.netloc: - self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId')) - self.story.addToList("category","The Manor") - else: - self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId')) - - # Each adapter needs to have a unique site abbreviation. - self.story.setMetadata('siteabbrev','gech') - - # The date format will vary from site to site. - # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - self.dateformat = "%d/%b/%Y" - - @staticmethod # must be @staticmethod, don't remove it. - def getSiteDomain(): - # The site domain. Does have www here, if it uses it. - return 'grangerenchanted.com' - - @classmethod - def getAcceptDomains(cls): - return ['grangerenchanted.com','malfoymanor.grangerenchanted.com'] - - @classmethod - def getSiteExampleURLs(cls): - return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234" - - def getSiteURLPattern(self): - return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$" - - ## Login seems to be reasonably standard across eFiction sites. - def needToLoginCheck(self, data): - if 'Registered Users Only' in data \ - or 'There is no such account on our website' in data \ - or "That password doesn't match the one in our database" in data: - return True - else: - return False - - def performLogin(self, url): - params = {} - - if self.password: - params['penname'] = self.username - params['password'] = self.password - else: - params['penname'] = self.getConfig("username") - params['password'] = self.getConfig("password") - params['cookiecheck'] = '1' - params['submit'] = 'Submit' - - if "enchant" in self.section: - loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login' - else: - loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login' - logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, - params['penname'])) - - d = self._fetchUrl(loginUrl, params) - - if "Member Account" not in d : #Member Account - logger.info("Failed to login to URL %s as %s" % (loginUrl, - params['penname'])) - raise exceptions.FailedToLogin(url,params['penname']) - return False - else: - return True - - ## Getting the chapter list and the meta data, plus 'is adult' checking. - def extractChapterUrlsAndMetadata(self): - - if self.is_adult or self.getConfig("is_adult"): - # Weirdly, different sites use different warning numbers. - # If the title search below fails, there's a good chance - # you need a different number. print data at that point - # and see what the 'click here to continue' url says. - addurl = "&ageconsent=ok&warning=1" - else: - addurl="" - - # index=1 makes sure we see the story chapter index. Some - # sites skip that for one-chapter stories. - url = self.url+addurl - logger.debug("URL: "+url) - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - - if self.needToLoginCheck(data): - # need to log in for this one. - self.performLogin(url) - data = self._fetchUrl(url) - - m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) - if m != None: - if self.is_adult or self.getConfig("is_adult"): - # We tried the default and still got a warning, so - # let's pull the warning number from the 'continue' - # link and reload data. - addurl = m.group(1) - # correct stupid & error in url. - addurl = addurl.replace("&","&") - url = self.url+'&index=1'+addurl - logger.debug("URL 2nd try: "+url) - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - else: - raise exceptions.AdultCheckRequired(self.url) - - if "Access denied. This story has not been validated by the adminstrators of this site." in data: - raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") - - # use BeautifulSoup HTML parser to make everything easier to find. - soup = self.make_soup(data) - # print data - - # Now go hunting for all the meta data and the chapter list. - - ## Title - a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',stripHTML(a)) - - # Find authorid and URL from... author url. - a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) - self.story.setMetadata('authorId',a['href'].split('=')[1]) - self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) - self.story.setMetadata('author',a.string) - - # Find the chapters: - for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): - # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl)) - - self.story.setMetadata('numChapters',len(self.chapterUrls)) - - # eFiction sites don't help us out a lot with their meta data - # formating, so it's a little ugly. - - # utility method - def defaultGetattr(d,k): - try: - return d[k] - except: - return "" - - # Rated: NC-17
etc - labels = soup.findAll('span',{'class':'label'}) - for labelspan in labels: - value = labelspan.nextSibling - label = labelspan.string - - if 'Summary' in label: - ## Everything until the next span class='label' - svalue = "" - while value and 'label' not in defaultGetattr(value,'class') and '' not in unicode(value): - svalue += unicode(value) - value = value.nextSibling - self.setDescription(url,svalue) - - if 'Rated' in label: - self.story.setMetadata('rating', value) - - if 'Word count' in label: - self.story.setMetadata('numWords', value) - - if 'Read' in label: - self.story.setMetadata('read', value) - - if 'Categories' in label: - cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) - for cat in cats: - self.story.addToList('category',cat.string) - - if 'Characters' in label: - chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) - for char in chars: - self.story.addToList('characters',char.string) - - if 'Genre' in label: - genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4')) - for genre in genres: - self.story.addToList('genre',genre.string) - - if 'Warnings' in label: - warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) - for warning in warnings: - self.story.addToList('warnings',warning.string) - - if 'Completed' in label: - if 'Yes' in value: - self.story.setMetadata('status', 'Completed') - else: - self.story.setMetadata('status', 'In-Progress') - - if 'Published' in label: - self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) - - if 'Updated' in label: - self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) - - try: - # Find Series name from series URL. - a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) - series_name = a.string - series_url = 'http://'+self.host+'/'+self.section+'/'+a['href'] - - # use BeautifulSoup HTML parser to make everything easier to find. - seriessoup = self.make_soup(self._fetchUrl(series_url)) - # can't use ^viewstory...$ in case of higher rated stories with javascript href. - storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) - i=1 - for a in storyas: - # skip 'report this' and 'TOC' links - if 'contact.php' not in a['href'] and 'index' not in a['href']: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - self.story.setMetadata('seriesUrl',series_url) - break - i+=1 - except: - # I find it hard to care if the series parsing fails - pass - - try: - self.story.setMetadata('reviews', - stripHTML(soup.find('div',{'id':'sort'}). - findAll('a', href=re.compile(r'^reviews.php'))[1])) - except: - # I find it hard to care if the series parsing fails - pass - - # grab the text for an individual chapter. - def getChapterText(self, url): - - logger.debug('Getting chapter text from: %s' % url) - - soup = self.make_soup(self._fetchUrl(url)) - - div = soup.find('div', {'id' : 'story1'}) - - if None == div: - raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - - return self.utf8FromSoup(url,div) diff --git a/fanficfare/adapters/adapter_nickandgregnet.py b/fanficfare/adapters/adapter_nickandgregnet.py deleted file mode 100644 index ffec118..0000000 --- a/fanficfare/adapters/adapter_nickandgregnet.py +++ /dev/null @@ -1,179 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Software: eFiction -import time -import logging -logger = logging.getLogger(__name__) -import re -import urllib2 - - -from ..htmlcleanup import stripHTML -from .. import exceptions as exceptions - -from base_adapter import BaseSiteAdapter, makeDate - -def getClass(): - return NickAndGregNetAdapter - -# Class name has to be unique. Our convention is camel case the -# sitename with Adapter at the end. www is skipped. -class NickAndGregNetAdapter(BaseSiteAdapter): - - def __init__(self, config, url): - BaseSiteAdapter.__init__(self, config, url) - - self.decode = ["Windows-1252", - "utf8"] # 1252 is a superset of iso-8859-1. - # Most sites that claim to be - # iso-8859-1 (and some that claim to be - # utf8) are really windows-1252. - self.username = "NoneGiven" # if left empty, site doesn't return any message at all. - self.password = "" - self.is_adult=False - - # get storyId from url--url validation guarantees query is only sid=1234 - self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) - - - # normalized story URL. - # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. - self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId')) - - # Each adapter needs to have a unique site abbreviation. - self.story.setMetadata('siteabbrev','nag') - - # The date format will vary from site to site. - # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - self.dateformat = "%Y/%m/%d" - - @staticmethod # must be @staticmethod, don't remove it. - def getSiteDomain(): - # The site domain. Does have www here, if it uses it. - return 'www.nickngreg.nl' - - @classmethod - def getAcceptDomains(cls): - return ['www.nickngreg.nl','www.nickandgreg.net'] - - @classmethod - def getSiteExampleURLs(cls): - return "http://"+cls.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234" - - def getSiteURLPattern(self): - return "http://("+self.getSiteDomain()+"|www.nickandgreg.net)"+re.escape("/desert_archive/viewstory.php?sid=")+r"\d+$" - - - ## Getting the chapter list and the meta data, plus 'is adult' checking. - def extractChapterUrlsAndMetadata(self): - - # index=1 makes sure we see the story chapter index. Some - # sites skip that for one-chapter stories. - url = self.url+'&i=1' - logger.debug("URL: "+url) - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - - if "Access denied. This story has not been validated by the adminstrators of this site." in data: - raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") - - # use BeautifulSoup HTML parser to make everything easier to find. - soup = self.make_soup(data) - # print data - - # Now go hunting for all the meta data and the chapter list. - - ## Title - a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',stripHTML(a)) - - # Find authorid and URL from... author url. - a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) - self.story.setMetadata('authorId',a['href'].split('=')[1]) - self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href']) - self.story.setMetadata('author',a.string) - - # Find the chapters: - chapters = soup.find('select') - for chapter in chapters.findAll('option'): - if chapter.text != 'Story Index' and chapter.text != 'Chapters': - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value'])) - - self.story.setMetadata('numChapters',len(self.chapterUrls)) - - asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl'))) - - for div in asoup.findAll('td', {'class' : 'tblborder6'}): - a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - if a != None: - break - - self.setDescription(url,div.find('br').nextSibling) - - a=div.text.split('Rating:') - if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0]) - - a=div.text.split('Characters:') - if len(a) == 2: - for char in a[1].split(' -')[0].split(', '): - self.story.addToList('characters',char) - - a=div.text.split('Genres:') - if len(a) == 2: - for genre in a[1].split(' -')[0].split(', '): - self.story.addToList('genre',genre) - - a=div.text.split('Warnings:') - if len(a) == 2: - for warn in a[1].split(' -')[0].split(', '): - if 'none' not in warn: - self.story.addToList('warnings',warn) - - a=div.text.split('Completed:') - if len(a) ==2: - if 'Yes' in a[1]: - self.story.setMetadata('status', 'Completed') - else: - self.story.setMetadata('status', 'In-Progress') - - a=div.text.split('Published:') - if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat)) - - a=div.text.split('Updated:') - if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat)) - - - # grab the text for an individual chapter. - def getChapterText(self, url): - - logger.debug('Getting chapter text from: %s' % url) - - soup = self.make_soup(self._fetchUrl(url)) - - # wrap a div around it. - divsoup = self.make_soup('
') - div = divsoup.find('div') - div.append(soup.find('table', {'class' : 'tblborder6'})) - - return self.utf8FromSoup(url,div) diff --git a/fanficfare/adapters/adapter_rabidreadercom.py b/fanficfare/adapters/adapter_rabidreadercom.py deleted file mode 100644 index 40025a7..0000000 --- a/fanficfare/adapters/adapter_rabidreadercom.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2016 FanFicFare team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Software: eFiction -from base_efiction_adapter import BaseEfictionAdapter - -class RabidReaderComAdapter(BaseEfictionAdapter): - - @staticmethod - def getSiteDomain(): - return 'www.therabidreader.com' - - @classmethod - def getSiteAbbrev(self): - return 'rrcom' - - @classmethod - def getDateFormat(self): - return "%d %b %Y" - -def getClass(): - return RabidReaderComAdapter diff --git a/fanficfare/adapters/adapter_scarheadnet.py b/fanficfare/adapters/adapter_scarheadnet.py deleted file mode 100644 index d7d3557..0000000 --- a/fanficfare/adapters/adapter_scarheadnet.py +++ /dev/null @@ -1,299 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Software: eFiction -import time -import logging -logger = logging.getLogger(__name__) -import re -import urllib2 - - -from ..htmlcleanup import stripHTML -from .. import exceptions as exceptions - -from base_adapter import BaseSiteAdapter, makeDate - -def getClass(): - return ScarHeadNetAdapter - -# Class name has to be unique. Our convention is camel case the -# sitename with Adapter at the end. www is skipped. -class ScarHeadNetAdapter(BaseSiteAdapter): - - def __init__(self, config, url): - BaseSiteAdapter.__init__(self, config, url) - - self.decode = ["Windows-1252", - "utf8"] # 1252 is a superset of iso-8859-1. - # Most sites that claim to be - # iso-8859-1 (and some that claim to be - # utf8) are really windows-1252. - self.username = "NoneGiven" # if left empty, site doesn't return any message at all. - self.password = "" - self.is_adult=False - - # get storyId from url--url validation guarantees query is only sid=1234 - self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) - - - # normalized story URL. - self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) - - # Each adapter needs to have a unique site abbreviation. - self.story.setMetadata('siteabbrev','shn') - - # The date format will vary from site to site. - # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - self.dateformat = "%d/%m/%y" - - @staticmethod # must be @staticmethod, don't remove it. - def getSiteDomain(): - # The site domain. Does have www here, if it uses it. - return 'scarhead.net' - - @classmethod - def getSiteExampleURLs(cls): - return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" - - def getSiteURLPattern(self): - return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" - - ## Login seems to be reasonably standard across eFiction sites. - def needToLoginCheck(self, data): - if 'Registered Users Only' in data \ - or 'There is no such account on our website' in data \ - or "That password doesn't match the one in our database" in data: - return True - else: - return False - - def performLogin(self, url): - params = {} - - if self.password: - params['penname'] = self.username - params['password'] = self.password - else: - params['penname'] = self.getConfig("username") - params['password'] = self.getConfig("password") - params['cookiecheck'] = '1' - params['submit'] = 'Submit' - - loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' - logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, - params['penname'])) - - d = self._fetchUrl(loginUrl, params) - - if "Member Account" not in d : #Member Account - logger.info("Failed to login to URL %s as %s" % (loginUrl, - params['penname'])) - raise exceptions.FailedToLogin(url,params['penname']) - return False - else: - return True - - ## Getting the chapter list and the meta data, plus 'is adult' checking. - def extractChapterUrlsAndMetadata(self): - - if self.is_adult or self.getConfig("is_adult"): - # Weirdly, different sites use different warning numbers. - # If the title search below fails, there's a good chance - # you need a different number. print data at that point - # and see what the 'click here to continue' url says. - addurl = "&ageconsent=ok&warning=5" - else: - addurl="" - - # index=1 makes sure we see the story chapter index. Some - # sites skip that for one-chapter stories. - url = self.url+'&index=1'+addurl - logger.debug("URL: "+url) - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - - if self.needToLoginCheck(data): - # need to log in for this one. - self.performLogin(url) - data = self._fetchUrl(url) - - # Since the warning text can change by warning level, let's - # look for the warning pass url. ksarchive uses - # &warning= -- actually, so do other sites. Must be an - # eFiction book. - - # viewstory.php?sid=1882&warning=4 - # viewstory.php?sid=1654&ageconsent=ok&warning=5 - #print data - #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) - m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) - if m != None: - if self.is_adult or self.getConfig("is_adult"): - # We tried the default and still got a warning, so - # let's pull the warning number from the 'continue' - # link and reload data. - addurl = m.group(1) - # correct stupid & error in url. - addurl = addurl.replace("&","&") - url = self.url+'&index=1'+addurl - logger.debug("URL 2nd try: "+url) - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - else: - raise exceptions.AdultCheckRequired(self.url) - - if "Access denied. This story has not been validated by the adminstrators of this site." in data: - raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") - - # use BeautifulSoup HTML parser to make everything easier to find. - soup = self.make_soup(data) - # print data - - # Now go hunting for all the meta data and the chapter list. - - pagetitle = soup.find('tr',{'valign':'top'}) - - ## Title - a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',stripHTML(a)) - - # Find authorid and URL from... author url. - a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) - self.story.setMetadata('authorId',a['href'].split('=')[1]) - self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) - self.story.setMetadata('author',a.string) - - # Find the chapters: - for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): - # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) - - self.story.setMetadata('numChapters',len(self.chapterUrls)) - - # eFiction sites don't help us out a lot with their meta data - # formating, so it's a little ugly. - - cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories')) - for cat in cats: - if '/' == cat.string[0]: - self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0]) - elif 'Harry' in cat.string: - self.story.addToList('ships',cat.string.split('(')[0]) - else: - self.story.addToList('category',cat.string) - if '(' in cat.string: - self.story.addToList('category',cat.string.split('(')[1].split(')')[0]) - - - - - chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters')) - for char in chars: - self.story.addToList('characters',char.string) - - genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) - for genre in genres: - self.story.addToList('genre',genre.string) - - warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) - for warning in warnings: - self.story.addToList('warnings',warning.string) - - textsoup = stripHTML(soup) - - a = textsoup.split('Published: ')[1].split(' ')[0] - self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat)) - a = textsoup.split('Updated: ')[1].split(' ')[0] - self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat)) - a = textsoup.split('Rating: ')[1].split(' ')[0] - self.story.setMetadata('rating', a) - a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0] - self.story.setMetadata('numWords', a) - a = textsoup.split('Completed: ')[1].split(' ')[0] - if 'Yes' in a: - self.story.setMetadata('status', 'Completed') - else: - self.story.setMetadata('status', 'In-Progress') - #a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0] - #self.setDescription(url,a) - - - - a=soup.find(text=re.compile("Summary: ")) - i=0 - svalue = "" - while i == 0: - try: - b = unicode(a) - svalue += b.split('Summary: ')[1] - except: - svalue += unicode(a) - if a.nextSibling != None: - a = a.nextSibling - else: - a = a.parent.nextSibling - if 'Disclaimer: ' in stripHTML(a): - i=1 - self.setDescription(url,svalue) - - try: - # Find Series name from series URL. - a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) - series_name = a.string - series_url = 'http://'+self.host+'/'+a['href'] - - # use BeautifulSoup HTML parser to make everything easier to find. - seriessoup = self.make_soup(self._fetchUrl(series_url)) - storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) - i=1 - for a in storyas: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - self.story.setMetadata('seriesUrl',series_url) - break - i+=1 - - except: - # I find it hard to care if the series parsing fails - pass - - # grab the text for an individual chapter. - def getChapterText(self, url): - - logger.debug('Getting chapter text from: %s' % url) - - soup = self.make_soup(self._fetchUrl(url)) - - div = soup.find('div', {'id' : 'story'}) - - if None == div: - raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - - return self.utf8FromSoup(url,div) diff --git a/fanficfare/adapters/adapter_tokrafandomnetcom.py b/fanficfare/adapters/adapter_tokrafandomnetcom.py deleted file mode 100644 index 2e6f054..0000000 --- a/fanficfare/adapters/adapter_tokrafandomnetcom.py +++ /dev/null @@ -1,238 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Software: eFiction -import time -import logging -logger = logging.getLogger(__name__) -import re -import urllib2 - - -from ..htmlcleanup import stripHTML -from .. import exceptions as exceptions - -from base_adapter import BaseSiteAdapter, makeDate - -def getClass(): - return TokraFandomnetComAdapter - -# Class name has to be unique. Our convention is camel case the -# sitename with Adapter at the end. www is skipped. -class TokraFandomnetComAdapter(BaseSiteAdapter): - - def __init__(self, config, url): - BaseSiteAdapter.__init__(self, config, url) - - self.decode = ["Windows-1252", - "utf8"] # 1252 is a superset of iso-8859-1. - # Most sites that claim to be - # iso-8859-1 (and some that claim to be - # utf8) are really windows-1252. - self.username = "NoneGiven" # if left empty, site doesn't return any message at all. - self.password = "" - self.is_adult=False - - # get storyId from url--url validation guarantees query is only sid=1234 - self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) - - - # normalized story URL. - self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) - - # Each adapter needs to have a unique site abbreviation. - self.story.setMetadata('siteabbrev','tokra') - - # The date format will vary from site to site. - # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - self.dateformat = "%m/%d/%Y" - - @staticmethod # must be @staticmethod, don't remove it. - def getSiteDomain(): - # The site domain. Does have www here, if it uses it. But it - # doesn't matter too much anymore. - return 'tokra.fandomnet.com' - - @classmethod - def getSiteExampleURLs(cls): - return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" - - def getSiteURLPattern(self): - return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" - - ## Getting the chapter list and the meta data, plus 'is adult' checking. - def extractChapterUrlsAndMetadata(self): - - if self.is_adult or self.getConfig("is_adult"): - # Weirdly, different sites use different warning numbers. - # If the title search below fails, there's a good chance - # you need a different number. print data at that point - # and see what the 'click here to continue' url says. - addurl = "&ageconsent=ok&warning=3" - else: - addurl="" - - # index=1 makes sure we see the story chapter index. Some - # sites skip that for one-chapter stories. - url = self.url+'&index=1'+addurl - logger.debug("URL: "+url) - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - - m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) - if m != None: - if self.is_adult or self.getConfig("is_adult"): - # We tried the default and still got a warning, so - # let's pull the warning number from the 'continue' - # link and reload data. - addurl = m.group(1) - # correct stupid & error in url. - addurl = addurl.replace("&","&") - url = self.url+'&index=1'+addurl - logger.debug("URL 2nd try: "+url) - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - else: - raise exceptions.AdultCheckRequired(self.url) - - if "Access denied. This story has not been validated by the adminstrators of this site." in data: - raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") - - # use BeautifulSoup HTML parser to make everything easier to find. - soup = self.make_soup(data) - #print data - - # Now go hunting for all the meta data and the chapter list. - - ## Title - a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',stripHTML(a)) - - # Find authorid and URL from... author url. - a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) - self.story.setMetadata('authorId',a['href'].split('=')[1]) - self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) - self.story.setMetadata('author',a.string) - - # Rating - rate = stripHTML(soup.find('div',{'id':'pagetitle'})) - rate = rate[rate.rindex('[')+1:rate.rindex(']')] - self.story.setMetadata('rating', rate) - - # Find the chapters: - for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): - # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) - - self.story.setMetadata('numChapters',len(self.chapterUrls)) - - # eFiction sites don't help us out a lot with their meta data - # formating, so it's a little ugly. - - metadiv = soup.find('div',{'class':'content'}) - smalldiv = metadiv.find('div',{'class':'small'}) - - # tokra categories -> genre - # categories will be filled from ini. - genres = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) - for genre in genres: - self.story.addToList('genre',genre.string) - - chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) - for char in chars: - self.story.addToList('characters',char.string) - - metatext = stripHTML(smalldiv) - - if 'Completed: Yes' in metatext: - self.story.setMetadata('status', 'Completed') - else: - self.story.setMetadata('status', 'In-Progress') - - wordstart=metatext.rindex('Word count:')+12 - words = metatext[wordstart:metatext.index(' ',wordstart)] - self.story.setMetadata('numWords', words) - - datesdiv = soup.find('div',{'class':'bottom'}) - dates = stripHTML(datesdiv).split() - # Published: 04/26/2011 Updated: 03/06/2013 - self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat)) - self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat)) - - try: - # Find Series name from series URL. - a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) - series_name = a.string - series_url = 'http://'+self.host+'/'+a['href'] - - # use BeautifulSoup HTML parser to make everything easier to find. - seriessoup = self.make_soup(self._fetchUrl(series_url)) - # can't use ^viewstory...$ in case of higher rated stories with javascript href. - storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) - i=1 - for a in storyas: - # skip 'report this' and 'TOC' links - if 'contact.php' not in a['href'] and 'index' not in a['href']: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - self.story.setMetadata('seriesUrl',series_url) - break - i+=1 - - except: - # I find it hard to care if the series parsing fails - pass - - # remove 'small' leaving only summary. - smalldiv.extract() - self.setDescription(url,metadiv) - - # grab the text for an individual chapter. - def getChapterText(self, url): - - logger.debug('Getting chapter text from: %s' % url) - - soup = self.make_soup(self._fetchUrl(url)) - - div = soup.find('div', {'class' : 'content'}) - - if None == div: - raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - - # remove some decorations while keeping notes. - remove = div.find('div', {'id' : 'pagetitle'}) - remove.extract() - - for remove in div.findAll('div', {'class' : 'right'}): - remove.extract() - - for remove in div.findAll('div', {'class' : 'left'}): - remove.extract() - - return self.utf8FromSoup(url,div)