mirror of
https://github.com/wassname/FanFicFare.git
synced 2026-09-10 11:40:24 +08:00
Remove support for dead sites.
This commit is contained in:
Jim Miller
@@ -85,7 +85,6 @@ import adapter_hpfanficarchivecom
|
||||
import adapter_twilightarchivescom
|
||||
import adapter_nhamagicalworldsus
|
||||
import adapter_hlfictionnet
|
||||
import adapter_grangerenchantedcom
|
||||
import adapter_dracoandginnycom
|
||||
import adapter_scarvesandcoffeenet
|
||||
import adapter_thepetulantpoetesscom
|
||||
@@ -102,13 +101,9 @@ import adapter_efictionestelielde
|
||||
import adapter_pommedesangcom
|
||||
import adapter_restrictedsectionorg
|
||||
import adapter_imagineeficcom
|
||||
import adapter_buffynfaithnet
|
||||
import adapter_psychficcom
|
||||
import adapter_tokrafandomnetcom
|
||||
import adapter_asr3slashzoneorg
|
||||
import adapter_nickandgregnet
|
||||
import adapter_potterheadsanonymouscom
|
||||
import adapter_scarheadnet
|
||||
import adapter_fictionpadcom
|
||||
import adapter_storiesonlinenet
|
||||
import adapter_trekiverseorg
|
||||
@@ -120,7 +115,6 @@ import adapter_nocturnallightnet
|
||||
import adapter_fanfichu
|
||||
import adapter_fanfictioncsodaidokhu
|
||||
import adapter_fictionmaniatv
|
||||
import adapter_bdsmgeschichten
|
||||
import adapter_tolkienfanfiction
|
||||
import adapter_themaplebookshelf
|
||||
import adapter_fannation
|
||||
@@ -139,13 +133,10 @@ import adapter_ninelivesarchivecom
|
||||
import adapter_masseffect2in
|
||||
import adapter_quotevcom
|
||||
import adapter_mcstoriescom
|
||||
|
||||
import adapter_lucifaelff
|
||||
import adapter_buffygilescom
|
||||
#import adapter_rubyquillcom
|
||||
import adapter_andromedawebcom # Not all lables are captured
|
||||
import adapter_andromedawebcom
|
||||
import adapter_artemisfowlcom
|
||||
import adapter_rabidreadercom
|
||||
import adapter_naiceanilmenet
|
||||
|
||||
## This bit of complexity allows adapters to be added by just adding
|
||||
|
||||
@@ -1,347 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2014 Fanficdownloader team, 2015 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import time
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import urllib2
|
||||
import urlparse
|
||||
import time
|
||||
|
||||
from bs4.element import Tag, Comment
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def _translate_date_german_english(date):
|
||||
fullmon = {"Januar":"01",
|
||||
"Februar":"02",
|
||||
u"März":"03",
|
||||
"April":"04",
|
||||
"Mai":"05",
|
||||
"Juni":"06",
|
||||
"Juli":"07",
|
||||
"August":"08",
|
||||
"September":"09",
|
||||
"Oktober":"10",
|
||||
"November":"11",
|
||||
"Dezember":"12"}
|
||||
for (name,num) in fullmon.items():
|
||||
date = date.replace(name,num)
|
||||
return date
|
||||
|
||||
_REGEX_TRAILING_DIGIT = re.compile("(\d+)$")
|
||||
_REGEX_DASH_TO_END = re.compile("-[^-]+$")
|
||||
_REGEX_CHAPTER_TITLE = re.compile(ur"""
|
||||
\s*
|
||||
[\u2013-]?
|
||||
\s*
|
||||
([\dIVX-]+)?
|
||||
\.?
|
||||
\s*
|
||||
[\[\(]?
|
||||
\s*
|
||||
(Teil|Kapitel|Tag)?
|
||||
\s*
|
||||
([\dIVX-]+)?
|
||||
\s*
|
||||
[\]\)]?
|
||||
\s*
|
||||
$
|
||||
""", re.VERBOSE)
|
||||
_INITIAL_STEP = 5
|
||||
|
||||
class BdsmGeschichtenAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["utf8", "Windows-1252"]
|
||||
|
||||
self.story.setMetadata('siteabbrev','bdsmgesch')
|
||||
|
||||
# Replace possible chapter numbering
|
||||
chapterMatch = _REGEX_TRAILING_DIGIT.search(url)
|
||||
if chapterMatch is None:
|
||||
self.maxChapter = 1
|
||||
else:
|
||||
self.maxChapter = int(chapterMatch.group(1))
|
||||
# url = re.sub(_REGEX_TRAILING_DIGIT, "1", url)
|
||||
|
||||
# set storyId
|
||||
self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
|
||||
|
||||
# normalize URL
|
||||
self._setURL('http://%s/%s' % (self.getSiteDomain(), self.story.getMetadata('storyId')))
|
||||
|
||||
self.dateformat = '%d. %m %Y - %H:%M'
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'bdsm-geschichten.net'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['www.bdsm-geschichten.net', 'www.bdsm-geschichten.net']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://www.bdsm-geschichten.net/title-of-story-1 http://bdsm-geschichten.net/title-of-story-1"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"http://(www\.)?bdsm-geschichten.net/(?P<storyId>[a-zA-Z0-9_-]+)"
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if not (self.is_adult or self.getConfig("is_adult")):
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
try:
|
||||
data1 = self._fetchUrl(self.url)
|
||||
soup = self.make_soup(data1)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))]
|
||||
|
||||
# Cache the soups so we won't have to redownload in getChapterText later
|
||||
self.soupsCache = {}
|
||||
self.soupsCache[self.url] = soup
|
||||
|
||||
# author
|
||||
authorDiv = soup.find("div", "author-pane-line author-name")
|
||||
authorId = authorDiv.string.strip()
|
||||
self.story.setMetadata('authorId', authorId)
|
||||
self.story.setMetadata('author', authorId)
|
||||
# TODO not really true need to be loggedin for this to work or fetch userid
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId)
|
||||
|
||||
# TODO better metadata
|
||||
date = soup.find("div", {"class": "submitted"}).string.strip()
|
||||
# 11. April 2015 - 17:08
|
||||
date = re.sub(r"(\d+\. \D+ \d+ - \d+:\d+).*", r"\1", date)
|
||||
date = _translate_date_german_english(date)
|
||||
self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
|
||||
title1 = soup.find("h1", {'class': 'title'}).string
|
||||
|
||||
|
||||
for tagLink in soup.find("ul", "taxonomy").findAll("a"):
|
||||
self.story.addToList('category', tagLink.string)
|
||||
|
||||
## Retrieve chapter soups
|
||||
if self.getConfig('find_chapters') == 'guess':
|
||||
self.chapterUrls = []
|
||||
self._find_chapters_by_guessing(title1)
|
||||
else:
|
||||
self._find_chapters_by_parsing(soup)
|
||||
|
||||
firstChapterUrl = self.chapterUrls[0][1]
|
||||
if firstChapterUrl in self.soupsCache:
|
||||
firstChapterSoup = self.soupsCache[firstChapterUrl]
|
||||
h1 = firstChapterSoup.find("h1").text
|
||||
else:
|
||||
h1 = soup.find("h1").text
|
||||
|
||||
h1 = re.sub(_REGEX_CHAPTER_TITLE, "", h1)
|
||||
self.story.setMetadata('title', h1)
|
||||
self.story.setMetadata('numChapters', len(self.chapterUrls))
|
||||
return
|
||||
|
||||
def _find_chapters_by_parsing(self, soup):
|
||||
|
||||
# store original soup
|
||||
origSoup = soup
|
||||
|
||||
#
|
||||
# find first chapter
|
||||
#
|
||||
firstLink = None
|
||||
firstLinkDiv = soup.find("div", "field-field-erster-teil")
|
||||
if firstLinkDiv is not None:
|
||||
firstLink = "http://%s%s" % (self.getSiteDomain(), firstLinkDiv.findNext("a")['href'])
|
||||
logger.debug("Found first chapter right away <%s>" % firstLink)
|
||||
try:
|
||||
soup = self.make_soup(self._fetchUrl(firstLink))
|
||||
self.soupsCache[firstLink] = soup
|
||||
self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise exceptions.StoryDoesNotExist(firstLink)
|
||||
else:
|
||||
logger.debug("DIDN'T find first chapter right away")
|
||||
# parse previous Link until first
|
||||
while True:
|
||||
prevLink = None
|
||||
prevLinkDiv = soup.find("div", "field-field-vorheriger-teil")
|
||||
if prevLinkDiv is not None:
|
||||
prevLink = prevLinkDiv.find("a")
|
||||
if prevLink is None:
|
||||
prevLink = soup.find("a", text=re.compile("<<<")) # <<<
|
||||
if prevLink is None:
|
||||
logger.debug("Couldn't find prev part")
|
||||
break
|
||||
else:
|
||||
logger.debug("Previous Chapter <%s>" % prevLink)
|
||||
if type(prevLink) != Tag or prevLink.name != "a":
|
||||
prevLink = prevLink.findParent("a")
|
||||
if prevLink is None or '#' in prevLink['href']:
|
||||
logger.debug("Couldn't find prev part (false positive) <%s>" % prevLink)
|
||||
break
|
||||
prevLink = prevLink['href']
|
||||
try:
|
||||
soup = self.make_soup(self._fetchUrl(prevLink))
|
||||
self.soupsCache[prevLink] = soup
|
||||
prevTtitle = soup.find("h1", {'class': 'title'}).string
|
||||
self.chapterUrls.insert(0, (prevTtitle, prevLink))
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(nextLink)
|
||||
else:
|
||||
raise e
|
||||
firstLink = prevLink
|
||||
|
||||
# if first chapter couldn't be determined, assume the URL originally
|
||||
# passed is the first chapter
|
||||
if firstLink is None:
|
||||
logger.debug("Couldn't set first chapter")
|
||||
firstLink = self.url
|
||||
self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
|
||||
|
||||
# set first URL
|
||||
logger.debug("Set first link: %s" % firstLink)
|
||||
self._setURL(firstLink)
|
||||
self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(firstLink).group('storyId'))
|
||||
|
||||
#
|
||||
# Parse next chapters
|
||||
#
|
||||
while True:
|
||||
nextLink = None
|
||||
nextLinkDiv = soup.find("div", "field-field-naechster-teil")
|
||||
if nextLinkDiv is not None:
|
||||
nextLink = nextLinkDiv.find("a")
|
||||
if nextLink is None:
|
||||
nextLink = soup.find("a", text=re.compile(">>>"))
|
||||
if nextLink is None:
|
||||
nextLink = soup.find("a", text=re.compile("Fortsetzung"))
|
||||
|
||||
if nextLink is None:
|
||||
logger.debug("Couldn't find next part")
|
||||
break
|
||||
else:
|
||||
if type(nextLink) != Tag or nextLink.name != "a":
|
||||
nextLink = nextLink.findParent("a")
|
||||
if nextLink is None or '#' in nextLink['href']:
|
||||
logger.debug("Couldn't find next part (false positive) <%s>" % nextLink)
|
||||
break
|
||||
nextLink = nextLink['href']
|
||||
|
||||
if not nextLink.startswith('http:'):
|
||||
nextLink = 'http://' + self.getSiteDomain() + nextLink
|
||||
|
||||
for loadedChapter in self.chapterUrls:
|
||||
if loadedChapter[0] == nextLink:
|
||||
logger.debug("ERROR: Repeating chapter <%s> Try to fix it" % nextLink)
|
||||
nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink)
|
||||
if nextLinkMatch is not None:
|
||||
curChap = nextLinkMatch.group(1)
|
||||
nextLink = re.sub(_REGEX_TRAILING_DIGIT, unicode(int(curChap) + 1), nextLink)
|
||||
else:
|
||||
break
|
||||
try:
|
||||
data = self._fetchUrl(nextLink)
|
||||
soup = self.make_soup(data)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(nextLink)
|
||||
else:
|
||||
raise e
|
||||
title2 = soup.find("h1", {'class': 'title'}).string
|
||||
self.chapterUrls.append((title2, nextLink))
|
||||
logger.debug("Grabbing next chapter URL " + nextLink)
|
||||
self.soupsCache[nextLink] = soup
|
||||
# [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))]
|
||||
logger.debug("Chapters: %s" % self.chapterUrls)
|
||||
|
||||
|
||||
def _find_chapters_by_guessing(self, title1):
|
||||
step = _INITIAL_STEP
|
||||
curMax = self.maxChapter + step
|
||||
lastHit = True
|
||||
while True:
|
||||
nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, unicode(curMax), self.url)
|
||||
if nextChapterUrl == self.url:
|
||||
logger.debug("Unable to guess next chapter because URL doesn't end in numbers")
|
||||
break;
|
||||
try:
|
||||
logger.debug("Trying chapter URL " + nextChapterUrl)
|
||||
data = self._fetchUrl(nextChapterUrl)
|
||||
hit = True
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
hit = False
|
||||
else:
|
||||
raise e
|
||||
if hit:
|
||||
logger.debug("Found chapter URL " + nextChapterUrl)
|
||||
self.maxChapter = curMax
|
||||
self.soupsCache[nextChapterUrl] = self.make_soup(data)
|
||||
if not lastHit:
|
||||
break
|
||||
lastHit = curMax
|
||||
curMax += step
|
||||
else:
|
||||
lastHit = False
|
||||
curMax -= 1
|
||||
logger.debug(curMax)
|
||||
|
||||
for i in xrange(1, self.maxChapter):
|
||||
nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, unicode(i), self.url)
|
||||
nextChapterTitle = re.sub("1", unicode(i), title1)
|
||||
self.chapterUrls.append((nextChapterTitle, nextChapterUrl))
|
||||
|
||||
def getChapterText(self, url):
|
||||
|
||||
if url in self.soupsCache:
|
||||
logger.debug('Getting chapter <%s> from cache' % url)
|
||||
soup = self.soupsCache[url]
|
||||
else:
|
||||
logger.debug('Downloading chapter <%s>' % url)
|
||||
data1 = self._fetchUrl(url)
|
||||
soup = self.make_soup(data1)
|
||||
#strip comments from soup
|
||||
[comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))]
|
||||
|
||||
# get story text
|
||||
storyDiv1 = soup.new_tag("div")
|
||||
for para in soup.find("div", "full-node").find('div', 'content').findAll("p"):
|
||||
storyDiv1.append(para)
|
||||
storyDiv1.append(soup.new_tag("br"))
|
||||
storytext = self.utf8FromSoup(url,storyDiv1)
|
||||
|
||||
return storytext
|
||||
|
||||
|
||||
def getClass():
|
||||
return BdsmGeschichtenAdapter
|
||||
@@ -1,292 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import time
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import urllib2
|
||||
import cookielib as cl
|
||||
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
|
||||
# This function is called by the downloader in all adapter_*.py files
|
||||
# in this dir to register the adapter class. So it needs to be
|
||||
# updated to reflect the class below it. That, plus getSiteDomain()
|
||||
# take care of 'Registering'.
|
||||
def getClass():
|
||||
return BuffyNFaithNetAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class BuffyNFaithNetAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.setHeader()
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query correct
|
||||
m = re.match(self.getSiteURLPattern(),url)
|
||||
if m:
|
||||
self.story.setMetadata('storyId',m.group('id'))
|
||||
|
||||
# normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
|
||||
nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
|
||||
self._setURL(nurl)
|
||||
else:
|
||||
raise exceptions.InvalidStoryURL(url,
|
||||
self.getSiteDomain(),
|
||||
self.getSiteExampleURLs())
|
||||
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','bnfnet')
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'buffynfaith.net'
|
||||
|
||||
@classmethod
|
||||
def stripURLParameters(cls,url):
|
||||
"Only needs to be overriden if URL contains more than one parameter"
|
||||
## This adapter needs at least two parameters left on the URL, act and id
|
||||
return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url)
|
||||
|
||||
def setHeader(self):
|
||||
"buffynfaith.net wants a Referer for images. Used both above and below(after cookieproc added)"
|
||||
self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=ovr&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234&ch=2"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
#http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
|
||||
#http://buffynfaith.net/fanfictions/index.php?act=vie&id=949
|
||||
#http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2
|
||||
p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\
|
||||
r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
|
||||
return p
|
||||
|
||||
def use_pagecache(self):
|
||||
'''
|
||||
adapters that will work with the page cache need to implement
|
||||
this and change it to True.
|
||||
'''
|
||||
return True
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
dateformat = "%d %B %Y"
|
||||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
#set a cookie to get past adult check
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
cookie = cl.Cookie(version=0, name='my_age', value='yes',
|
||||
port=None, port_specified=False,
|
||||
domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
|
||||
path='/', path_specified=True,
|
||||
secure=False,
|
||||
expires=time.time()+10000,
|
||||
discard=False,
|
||||
comment=None,
|
||||
comment_url=None,
|
||||
rest={'HttpOnly': None},
|
||||
rfc2109=False)
|
||||
self.get_cookiejar().set_cookie(cookie)
|
||||
self.setHeader()
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
#print data
|
||||
|
||||
if "ADULT CONTENT WARNING" in data:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = self.make_soup(data)
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
#stuff in <head>: description
|
||||
svalue = soup.head.find('meta',attrs={'name':'description'})['content']
|
||||
#self.story.setMetadata('description',svalue)
|
||||
self.setDescription(url,svalue)
|
||||
|
||||
#useful stuff in rest of doc, all contained in this:
|
||||
doc = soup.body.find('div', id='my_wrapper')
|
||||
|
||||
#first the site category (more of a genre to me, meh) and title, in this element:
|
||||
mt = doc.find('div',attrs={'class':'maintitle'})
|
||||
self.story.addToList('genre',mt.findAll('a')[1].string)
|
||||
self.story.setMetadata('title',stripHTML(mt).split(u'»')[-1].strip())
|
||||
del mt
|
||||
|
||||
#the actual category, for me, is 'Buffy: The Vampire Slayer'
|
||||
#self.story.addToList('category','Buffy: The Vampire Slayer')
|
||||
#No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini
|
||||
|
||||
#then a block that sits in a table cell like so:
|
||||
#(contains a lot of metadata)
|
||||
mblock = doc.find('td', align='left', width = '70%').contents
|
||||
while len(mblock) > 0:
|
||||
i = mblock.pop(0)
|
||||
if 'Author:' in i.string:
|
||||
#drop empty space
|
||||
mblock.pop(0)
|
||||
#get author link
|
||||
a = mblock.pop(0)
|
||||
authre = re.escape('./index.php?act=bio&id=')+'(?P<authid>\d+)'
|
||||
m = re.match(authre,a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
self.story.setMetadata('authorId',m.group('authid'))
|
||||
authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(),
|
||||
self.story.getMetadata('authorId'))
|
||||
self.story.setMetadata('authorUrl',authurl,condremoveentities=False)
|
||||
#drop empty space
|
||||
mblock.pop(0)
|
||||
if 'Rating:' in i.string:
|
||||
self.story.setMetadata('rating',mblock.pop(0).strip())
|
||||
if 'Published:' in i.string:
|
||||
date = mblock.pop(0).strip()
|
||||
#get rid of 'st', 'nd', 'rd', 'th' after day number
|
||||
date = date[0:2]+date[4:]
|
||||
self.story.setMetadata('datePublished',makeDate(date, dateformat))
|
||||
if 'Last Updated:' in i.string:
|
||||
date = mblock.pop(0).strip()
|
||||
#get rid of 'st', 'nd', 'rd', 'th' after day number
|
||||
date = date[0:2]+date[4:]
|
||||
self.story.setMetadata('dateUpdated',makeDate(date, dateformat))
|
||||
if 'Genre:' in i.string:
|
||||
genres = mblock.pop(0).strip()
|
||||
genres = genres.split('/')
|
||||
for genre in genres: self.story.addToList('genre',genre)
|
||||
#end ifs
|
||||
#end while
|
||||
|
||||
# Find the chapter selector
|
||||
select = soup.find('select', { 'name' : 'ch' } )
|
||||
|
||||
if select is None:
|
||||
# no selector found, so it's a one-chapter story.
|
||||
#self.chapterUrls.append((self.story.getMetadata('title'),url))
|
||||
self.chapterUrls.append((self.story.getMetadata('title'),url))
|
||||
else:
|
||||
allOptions = select.findAll('option')
|
||||
for o in allOptions:
|
||||
url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(),
|
||||
self.story.getMetadata('storyId'),
|
||||
o['value'])
|
||||
title = u"%s" % o
|
||||
title = stripHTML(title)
|
||||
ts = title.split(' ',1)
|
||||
title = ts[0]+'. '+ts[1]
|
||||
self.chapterUrls.append((title,url))
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
## Go scrape the rest of the metadata from the author's page.
|
||||
data = self._fetchUrl(self.story.getMetadata('authorUrl'))
|
||||
soup = self.make_soup(data)
|
||||
#find the story link and its parent div
|
||||
storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
|
||||
storydiv = storya.parent
|
||||
#warnings come under a <spawn> tag. Never seen that before...
|
||||
#appears to just be a line of freeform text, not necessarily a list
|
||||
#optional
|
||||
spawn = storydiv.find('spawn',{'id':'warnings'})
|
||||
if spawn is not None:
|
||||
warns = spawn.nextSibling.strip()
|
||||
self.story.addToList('warnings',warns)
|
||||
#some meta in spans - this should get all, even the ones jammed in a table
|
||||
spans = storydiv.findAll('span')
|
||||
for s in spans:
|
||||
if s.string == 'Ship:':
|
||||
list = s.nextSibling.strip().split()
|
||||
self.story.extendList('ships',list)
|
||||
if s.string == 'Characters:':
|
||||
list = s.nextSibling.strip().split(',')
|
||||
self.story.extendList('characters',list)
|
||||
if s.string == 'Status:':
|
||||
st = s.nextSibling.strip()
|
||||
self.story.setMetadata('status',st)
|
||||
if s.string == 'Words:':
|
||||
st = s.nextSibling.strip()
|
||||
self.story.setMetadata('numWords',st)
|
||||
|
||||
#reviews - is this worth having?
|
||||
#ffnet adapter gathers it, don't know if anything else does
|
||||
#or if it's ever going to be used!
|
||||
a = storydiv.find('a',{'id':'bold-blue'})
|
||||
if a:
|
||||
revs = a.nextSibling.strip()[1:-1]
|
||||
self.story.setMetadata('reviews',st)
|
||||
else:
|
||||
revs = '0'
|
||||
self.story.setMetadata('reviews',st)
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self._fetchUrl(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'fanfiction'})
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
#remove all the unnecessary bookmark tags
|
||||
[s.extract() for s in div('div',{'class':"tiny_box2"})]
|
||||
|
||||
#is there a review link?
|
||||
r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$"))
|
||||
if r is not None:
|
||||
#remove the review link and its parent div
|
||||
r.parent.extract()
|
||||
|
||||
#There might also be a link to the sequel on the last chapter
|
||||
#I'm inclined to keep it in, but the URL needs to be changed from relative to absolute
|
||||
#Shame there isn't proper series metadata available
|
||||
#(I couldn't find it anyway)
|
||||
s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$"))
|
||||
if s is not None:
|
||||
s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:]
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
@@ -1,309 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team, 2015 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
import time
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return GrangerEnchantedCom
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class GrangerEnchantedCom(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
self.section=self.parsedUrl.path.split('/',)[1]
|
||||
|
||||
# normalized story URL.
|
||||
if "malfoymanor" in self.parsedUrl.netloc:
|
||||
self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self.story.addToList("category","The Manor")
|
||||
else:
|
||||
self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','gech')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%d/%b/%Y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'grangerenchanted.com'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Registered Users Only' in data \
|
||||
or 'There is no such account on our website' in data \
|
||||
or "That password doesn't match the one in our database" in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def performLogin(self, url):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['penname'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['penname'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
params['cookiecheck'] = '1'
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
if "enchant" in self.section:
|
||||
loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
|
||||
else:
|
||||
loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
d = self._fetchUrl(loginUrl, params)
|
||||
|
||||
if "Member Account" not in d : #Member Account
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['penname']))
|
||||
raise exceptions.FailedToLogin(url,params['penname'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&ageconsent=ok&warning=1"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
# need to log in for this one.
|
||||
self.performLogin(url)
|
||||
data = self._fetchUrl(url)
|
||||
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while value and 'label' not in defaultGetattr(value,'class') and '<span class="label">' not in unicode(value):
|
||||
svalue += unicode(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Read' in label:
|
||||
self.story.setMetadata('read', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
seriessoup = self.make_soup(self._fetchUrl(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
try:
|
||||
self.story.setMetadata('reviews',
|
||||
stripHTML(soup.find('div',{'id':'sort'}).
|
||||
findAll('a', href=re.compile(r'^reviews.php'))[1]))
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self._fetchUrl(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story1'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
@@ -1,179 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
import time
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return NickAndGregNetAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class NickAndGregNetAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
# XXX Most sites don't have the /fanfic part. Replace all to remove it usually.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','nag')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%Y/%m/%d"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'www.nickngreg.nl'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['www.nickngreg.nl','www.nickandgreg.net']
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return "http://("+self.getSiteDomain()+"|www.nickandgreg.net)"+re.escape("/desert_archive/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&i=1'
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
chapters = soup.find('select')
|
||||
for chapter in chapters.findAll('option'):
|
||||
if chapter.text != 'Story Index' and chapter.text != 'Chapters':
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl')))
|
||||
|
||||
for div in asoup.findAll('td', {'class' : 'tblborder6'}):
|
||||
a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
if a != None:
|
||||
break
|
||||
|
||||
self.setDescription(url,div.find('br').nextSibling)
|
||||
|
||||
a=div.text.split('Rating:')
|
||||
if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
|
||||
|
||||
a=div.text.split('Characters:')
|
||||
if len(a) == 2:
|
||||
for char in a[1].split(' -')[0].split(', '):
|
||||
self.story.addToList('characters',char)
|
||||
|
||||
a=div.text.split('Genres:')
|
||||
if len(a) == 2:
|
||||
for genre in a[1].split(' -')[0].split(', '):
|
||||
self.story.addToList('genre',genre)
|
||||
|
||||
a=div.text.split('Warnings:')
|
||||
if len(a) == 2:
|
||||
for warn in a[1].split(' -')[0].split(', '):
|
||||
if 'none' not in warn:
|
||||
self.story.addToList('warnings',warn)
|
||||
|
||||
a=div.text.split('Completed:')
|
||||
if len(a) ==2:
|
||||
if 'Yes' in a[1]:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
a=div.text.split('Published:')
|
||||
if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
|
||||
|
||||
a=div.text.split('Updated:')
|
||||
if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
|
||||
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self._fetchUrl(url))
|
||||
|
||||
# wrap a div around it.
|
||||
divsoup = self.make_soup('<div class="story"></div>')
|
||||
div = divsoup.find('div')
|
||||
div.append(soup.find('table', {'class' : 'tblborder6'}))
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
@@ -1,36 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
from base_efiction_adapter import BaseEfictionAdapter
|
||||
|
||||
class RabidReaderComAdapter(BaseEfictionAdapter):
|
||||
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
return 'www.therabidreader.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteAbbrev(self):
|
||||
return 'rrcom'
|
||||
|
||||
@classmethod
|
||||
def getDateFormat(self):
|
||||
return "%d %b %Y"
|
||||
|
||||
def getClass():
|
||||
return RabidReaderComAdapter
|
||||
@@ -1,299 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
import time
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return ScarHeadNetAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class ScarHeadNetAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','shn')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%d/%m/%y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'scarhead.net'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Registered Users Only' in data \
|
||||
or 'There is no such account on our website' in data \
|
||||
or "That password doesn't match the one in our database" in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def performLogin(self, url):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['penname'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['penname'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
params['cookiecheck'] = '1'
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
d = self._fetchUrl(loginUrl, params)
|
||||
|
||||
if "Member Account" not in d : #Member Account
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['penname']))
|
||||
raise exceptions.FailedToLogin(url,params['penname'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&ageconsent=ok&warning=5"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
# need to log in for this one.
|
||||
self.performLogin(url)
|
||||
data = self._fetchUrl(url)
|
||||
|
||||
# Since the warning text can change by warning level, let's
|
||||
# look for the warning pass url. ksarchive uses
|
||||
# &warning= -- actually, so do other sites. Must be an
|
||||
# eFiction book.
|
||||
|
||||
# viewstory.php?sid=1882&warning=4
|
||||
# viewstory.php?sid=1654&ageconsent=ok&warning=5
|
||||
#print data
|
||||
#m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = self.make_soup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
pagetitle = soup.find('tr',{'valign':'top'})
|
||||
|
||||
## Title
|
||||
a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
if '/' == cat.string[0]:
|
||||
self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0])
|
||||
elif 'Harry' in cat.string:
|
||||
self.story.addToList('ships',cat.string.split('(')[0])
|
||||
else:
|
||||
self.story.addToList('category',cat.string)
|
||||
if '(' in cat.string:
|
||||
self.story.addToList('category',cat.string.split('(')[1].split(')')[0])
|
||||
|
||||
|
||||
|
||||
|
||||
chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
textsoup = stripHTML(soup)
|
||||
|
||||
a = textsoup.split('Published: ')[1].split(' ')[0]
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat))
|
||||
a = textsoup.split('Updated: ')[1].split(' ')[0]
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat))
|
||||
a = textsoup.split('Rating: ')[1].split(' ')[0]
|
||||
self.story.setMetadata('rating', a)
|
||||
a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0]
|
||||
self.story.setMetadata('numWords', a)
|
||||
a = textsoup.split('Completed: ')[1].split(' ')[0]
|
||||
if 'Yes' in a:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
#a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0]
|
||||
#self.setDescription(url,a)
|
||||
|
||||
|
||||
|
||||
a=soup.find(text=re.compile("Summary: "))
|
||||
i=0
|
||||
svalue = ""
|
||||
while i == 0:
|
||||
try:
|
||||
b = unicode(a)
|
||||
svalue += b.split('Summary: ')[1]
|
||||
except:
|
||||
svalue += unicode(a)
|
||||
if a.nextSibling != None:
|
||||
a = a.nextSibling
|
||||
else:
|
||||
a = a.parent.nextSibling
|
||||
if 'Disclaimer: ' in stripHTML(a):
|
||||
i=1
|
||||
self.setDescription(url,svalue)
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
seriessoup = self.make_soup(self._fetchUrl(series_url))
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self._fetchUrl(url))
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
@@ -1,238 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
import time
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return TokraFandomnetComAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class TokraFandomnetComAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','tokra')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%m/%d/%Y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it. But it
|
||||
# doesn't matter too much anymore.
|
||||
return 'tokra.fandomnet.com'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&ageconsent=ok&warning=3"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = self.make_soup(data)
|
||||
#print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Rating
|
||||
rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
|
||||
rate = rate[rate.rindex('[')+1:rate.rindex(']')]
|
||||
self.story.setMetadata('rating', rate)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
metadiv = soup.find('div',{'class':'content'})
|
||||
smalldiv = metadiv.find('div',{'class':'small'})
|
||||
|
||||
# tokra categories -> genre
|
||||
# categories will be filled from ini.
|
||||
genres = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
metatext = stripHTML(smalldiv)
|
||||
|
||||
if 'Completed: Yes' in metatext:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
wordstart=metatext.rindex('Word count:')+12
|
||||
words = metatext[wordstart:metatext.index(' ',wordstart)]
|
||||
self.story.setMetadata('numWords', words)
|
||||
|
||||
datesdiv = soup.find('div',{'class':'bottom'})
|
||||
dates = stripHTML(datesdiv).split()
|
||||
# Published: 04/26/2011 Updated: 03/06/2013
|
||||
self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
seriessoup = self.make_soup(self._fetchUrl(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
self.story.setMetadata('seriesUrl',series_url)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# remove 'small' leaving only summary.
|
||||
smalldiv.extract()
|
||||
self.setDescription(url,metadiv)
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = self.make_soup(self._fetchUrl(url))
|
||||
|
||||
div = soup.find('div', {'class' : 'content'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
# remove some decorations while keeping notes.
|
||||
remove = div.find('div', {'id' : 'pagetitle'})
|
||||
remove.extract()
|
||||
|
||||
for remove in div.findAll('div', {'class' : 'right'}):
|
||||
remove.extract()
|
||||
|
||||
for remove in div.findAll('div', {'class' : 'left'}):
|
||||
remove.extract()
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
Reference in New Issue
Block a user