mirror of
https://github.com/wassname/FanFicFare.git
synced 2026-09-11 11:52:07 +08:00
New site: www.efpfanfic.net
This commit is contained in:
Jim Miller
@@ -905,6 +905,21 @@ extracategories:InuYasha
|
||||
extracharacters:Sesshoumaru,Kagome
|
||||
extraships:Sesshoumaru/Kagome
|
||||
|
||||
[www.efpfanfic.net]
|
||||
## Some sites require login (or login for some rated stories) The
|
||||
## program can prompt you, or you can save it in config. In
|
||||
## commandline version, this should go in your personal.ini, not
|
||||
## defaults.ini.
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
## Extra metadata that this adapter knows about. See [dramione.org]
|
||||
## for examples of how to use them.
|
||||
extra_valid_entries:notes,context,type
|
||||
notes_label:Notes
|
||||
context_label:Context
|
||||
type_label:Type of Couple
|
||||
|
||||
[www.fanfiction.net]
|
||||
## fanfiction.net's 'cover' images are really just tiny thumbnails.
|
||||
## Change this to false to use them anyway.
|
||||
|
||||
@@ -105,6 +105,7 @@ import adapter_bloodtiesfancom
|
||||
import adapter_indeathnet
|
||||
import adapter_jlaunlimitedcom
|
||||
import adapter_qafficcom
|
||||
import adapter_efpfanficnet
|
||||
|
||||
|
||||
## This bit of complexity allows adapters to be added by just adding
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import time
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
from .. import BeautifulSoup as bs
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return EFPFanFicNet
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class EFPFanFicNet(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','efp')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%d/%m/%y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'www.efpfanfic.net'
|
||||
|
||||
def getSiteExampleURLs(self):
|
||||
return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Fai il login e leggi la storia!' in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def performLogin(self, url):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['penname'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['penname'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
params['cookiecheck'] = '1'
|
||||
params['submit'] = 'Invia'
|
||||
|
||||
loginUrl = 'http://' + self.getSiteDomain() + '/user.php?sid='+self.story.getMetadata('storyId')
|
||||
logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
d = self._fetchUrl(loginUrl, params)
|
||||
|
||||
if '<a class="menu" href="newaccount.php">' in d : # register for new account link
|
||||
logger.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['penname']))
|
||||
raise exceptions.FailedToLogin(url,params['penname'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
# need to log in for this one.
|
||||
self.performLogin(url)
|
||||
data = self._fetchUrl(url)
|
||||
|
||||
# if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
# raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = bs.BeautifulSoup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapter selector
|
||||
select = soup.find('select', { 'name' : 'sid' } )
|
||||
|
||||
if select is None:
|
||||
# no selector found, so it's a one-chapter story.
|
||||
self.chapterUrls.append((self.story.getMetadata('title'),url))
|
||||
else:
|
||||
allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
|
||||
for o in allOptions:
|
||||
url = u'http://%s/%s' % ( self.getSiteDomain(),
|
||||
o['value'])
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
title = stripHTML(o)
|
||||
self.chapterUrls.append((title,url))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# normalize story URL to first chapter if later chapter URL was given:
|
||||
url = self.chapterUrls[0][1].replace('&i=1','')
|
||||
logger.debug("Normalizing to URL: "+url)
|
||||
self._setURL(url)
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
storya = None
|
||||
authsoup = None
|
||||
authurl = self.story.getMetadata('authorUrl')
|
||||
|
||||
## author can have more than one page of stories.
|
||||
while storya == None:
|
||||
|
||||
# no storya, but do have authsoup--we're looping on author pages.
|
||||
if authsoup != None:
|
||||
# last author link with offset should be the 'next' link.
|
||||
authurl = u'http://%s/%s' % ( self.getSiteDomain(),
|
||||
authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
|
||||
|
||||
# Need author page for most of the metadata.
|
||||
logger.debug("fetching author page: (%s)"%authurl)
|
||||
authsoup = bs.BeautifulSoup(self._fetchUrl(authurl))
|
||||
#print("authsoup:%s"%authsoup)
|
||||
|
||||
storya = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
|
||||
|
||||
storyblock = storya.parent.parent.parent
|
||||
|
||||
self.setDescription(url,storyblock.find('div', {'class':'introbloc'}))
|
||||
|
||||
noteblock = storyblock.find('div', {'class':'notebloc'})
|
||||
#print("%s"%noteblock)
|
||||
notetext = ("%s" % noteblock).replace("<br />"," |")
|
||||
# <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />␍
|
||||
# Tipo di coppia: Het | Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio | Note: OOC | Avvertimenti: Tematiche delicate<br />␍
|
||||
# Categoria: <a href="categories.php?catid=1&parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&a=">3</a> recensioni</div>
|
||||
|
||||
cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
for item in notetext.split("|"):
|
||||
if ":" in item:
|
||||
(label,value) = item.split(":")
|
||||
label=label.strip()
|
||||
value=value.strip()
|
||||
else:
|
||||
label=value=item.strip()
|
||||
|
||||
if 'Pubblicata' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Aggiornata' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if label == "Completa":
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
|
||||
if label == "In corso":
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Rating' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Personaggi' in label:
|
||||
for val in value.split(","):
|
||||
self.story.addToList('characters',val)
|
||||
|
||||
if 'Genere' in label:
|
||||
for val in value.split(","):
|
||||
self.story.addToList('genre',val)
|
||||
|
||||
if 'Coppie' in label:
|
||||
for val in value.split(","):
|
||||
self.story.addToList('ships',val)
|
||||
|
||||
if 'Avvertimenti' in label:
|
||||
for val in value.split(","):
|
||||
if val != "None":
|
||||
self.story.addToList('warnings',val)
|
||||
|
||||
# 'extra' metadata for this adapter:
|
||||
|
||||
if 'Tipo di coppia' in label:
|
||||
for val in value.split(","):
|
||||
self.story.addToList('type',val)
|
||||
|
||||
if 'Note' in label:
|
||||
for val in value.split(","):
|
||||
if val != "None":
|
||||
self.story.addToList('notes',val)
|
||||
|
||||
if 'Contesto' in label:
|
||||
self.story.setMetadata('context', value)
|
||||
|
||||
## Note--efp doesn't provide word count.
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?ssid=\d+&i=1"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
|
||||
self.setSeries(series_name, i)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = bs.BeautifulSoup(self._fetchUrl(url))
|
||||
|
||||
div = soup.find('div', {'class' : 'storia'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
+8
-13
@@ -54,20 +54,11 @@
|
||||
much easier. </p>
|
||||
</div>
|
||||
<!-- put announcements here, h3 is a good title size. -->
|
||||
<h3>New Sites:</h3>
|
||||
<h3>New Site:</h3>
|
||||
<p>
|
||||
Two new sites thanks to besnef:
|
||||
New Italian language site:
|
||||
<ul>
|
||||
<li><a href="http://indeath.net">indeath.net</a></li>
|
||||
<li><a href="http://www.jlaunlimited.com">www.jlaunlimited.com</a></li>
|
||||
</ul>
|
||||
</p>
|
||||
<h3>New Fixes:</h3>
|
||||
<p>
|
||||
<ul>
|
||||
<li>Fixes for changes to fanfiktion.de formatting.</li>
|
||||
<li>Fix for bad author on twiwritenet with some skins.</li>
|
||||
<li>Change from www.ncisfiction.com to www.ncisfiction.net due to the ncisfiction.com domain expiring.</li>
|
||||
<li><a href="http://www.efpfanfic.net">www.efpfanfic.net</a></li>
|
||||
</ul>
|
||||
</p>
|
||||
<p>
|
||||
@@ -570,8 +561,12 @@
|
||||
Use the URL of the story's first chapter, such as
|
||||
<br /><a href="http://www.jlaunlimited.com/eFiction1.1/viewstory.php?sid=1234">http://www.jlaunlimited.com/eFiction1.1/viewstory.php?sid=1234</a>
|
||||
</dd>
|
||||
<dt>www.efpfanfic.net</dt>
|
||||
<dd>
|
||||
Use the URL of any story chapter, such as
|
||||
<br /><a href="http://www.efpfanfic.net/viewstory.php?sid=12345">http://www.efpfanfic.net/viewstory.php?sid=12345</a>
|
||||
</dd>
|
||||
</dl>
|
||||
|
||||
|
||||
<p>
|
||||
A few additional things to know, which will make your life substantially easier:
|
||||
|
||||
@@ -896,6 +896,21 @@ extracategories:InuYasha
|
||||
extracharacters:Sesshoumaru,Kagome
|
||||
extraships:Sesshoumaru/Kagome
|
||||
|
||||
[www.efpfanfic.net]
|
||||
## Some sites require login (or login for some rated stories) The
|
||||
## program can prompt you, or you can save it in config. In
|
||||
## commandline version, this should go in your personal.ini, not
|
||||
## defaults.ini.
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
## Extra metadata that this adapter knows about. See [dramione.org]
|
||||
## for examples of how to use them.
|
||||
extra_valid_entries:notes,context,type
|
||||
notes_label:Notes
|
||||
context_label:Context
|
||||
type_label:Type of Couple
|
||||
|
||||
[www.fanfiction.net]
|
||||
## fanfiction.net's 'cover' images are really just tiny thumbnails.
|
||||
## Change this to false to use them anyway.
|
||||
|
||||
Reference in New Issue
Block a user