diff --git a/defaults.ini b/defaults.ini index 0e74405..47bd261 100644 --- a/defaults.ini +++ b/defaults.ini @@ -905,6 +905,21 @@ extracategories:InuYasha extracharacters:Sesshoumaru,Kagome extraships:Sesshoumaru/Kagome +[www.efpfanfic.net] +## Some sites require login (or login for some rated stories) The +## program can prompt you, or you can save it in config. In +## commandline version, this should go in your personal.ini, not +## defaults.ini. +#username:YourName +#password:yourpassword + +## Extra metadata that this adapter knows about. See [dramione.org] +## for examples of how to use them. +extra_valid_entries:notes,context,type +notes_label:Notes +context_label:Context +type_label:Type of Couple + [www.fanfiction.net] ## fanfiction.net's 'cover' images are really just tiny thumbnails. ## Change this to false to use them anyway. diff --git a/fanficdownloader/adapters/__init__.py b/fanficdownloader/adapters/__init__.py index ce589f4..011c5b4 100644 --- a/fanficdownloader/adapters/__init__.py +++ b/fanficdownloader/adapters/__init__.py @@ -105,6 +105,7 @@ import adapter_bloodtiesfancom import adapter_indeathnet import adapter_jlaunlimitedcom import adapter_qafficcom +import adapter_efpfanficnet ## This bit of complexity allows adapters to be added by just adding diff --git a/fanficdownloader/adapters/adapter_efpfanficnet.py b/fanficdownloader/adapters/adapter_efpfanficnet.py new file mode 100644 index 0000000..55e60fd --- /dev/null +++ b/fanficdownloader/adapters/adapter_efpfanficnet.py @@ -0,0 +1,296 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return EFPFanFicNet + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class EFPFanFicNet(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + logger.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','efp') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%m/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.efpfanfic.net' + + def getSiteExampleURLs(self): + return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Fai il login e leggi la storia!' in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Invia' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?sid='+self.story.getMetadata('storyId') + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if '' in d : # register for new account link + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # if "Access denied. This story has not been validated by the adminstrators of this site." in data: + # raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapter selector + select = soup.find('select', { 'name' : 'sid' } ) + + if select is None: + # no selector found, so it's a one-chapter story. + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')}) + for o in allOptions: + url = u'http://%s/%s' % ( self.getSiteDomain(), + o['value']) + # just in case there's tags, like in chapter titles. + title = stripHTML(o) + self.chapterUrls.append((title,url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # normalize story URL to first chapter if later chapter URL was given: + url = self.chapterUrls[0][1].replace('&i=1','') + logger.debug("Normalizing to URL: "+url) + self._setURL(url) + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + logger.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + storya = None + authsoup = None + authurl = self.story.getMetadata('authorUrl') + + ## author can have more than one page of stories. + while storya == None: + + # no storya, but do have authsoup--we're looping on author pages. + if authsoup != None: + # last author link with offset should be the 'next' link. + authurl = u'http://%s/%s' % ( self.getSiteDomain(), + authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] ) + + # Need author page for most of the metadata. + logger.debug("fetching author page: (%s)"%authurl) + authsoup = bs.BeautifulSoup(self._fetchUrl(authurl)) + #print("authsoup:%s"%authsoup) + + storya = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$')) + + storyblock = storya.parent.parent.parent + + self.setDescription(url,storyblock.find('div', {'class':'introbloc'})) + + noteblock = storyblock.find('div', {'class':'notebloc'}) + #print("%s"%noteblock) + notetext = ("%s" % noteblock).replace("
"," |") + #
Autore: Cendrillon89 | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa
+ # Tipo di coppia: Het | Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio | Note: OOC | Avvertimenti: Tematiche delicate
+ # Categoria: Anime & Manga > Naruto | Contesto: Naruto Shippuuden | Leggi le 3 recensioni
+ + cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + for item in notetext.split("|"): + if ":" in item: + (label,value) = item.split(":") + label=label.strip() + value=value.strip() + else: + label=value=item.strip() + + if 'Pubblicata' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Aggiornata' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + if label == "Completa": + self.story.setMetadata('status', 'Completed') + + if label == "In corso": + self.story.setMetadata('status', 'In-Progress') + + if 'Rating' in label: + self.story.setMetadata('rating', value) + + if 'Personaggi' in label: + for val in value.split(","): + self.story.addToList('characters',val) + + if 'Genere' in label: + for val in value.split(","): + self.story.addToList('genre',val) + + if 'Coppie' in label: + for val in value.split(","): + self.story.addToList('ships',val) + + if 'Avvertimenti' in label: + for val in value.split(","): + if val != "None": + self.story.addToList('warnings',val) + + # 'extra' metadata for this adapter: + + if 'Tipo di coppia' in label: + for val in value.split(","): + self.story.addToList('type',val) + + if 'Note' in label: + for val in value.split(","): + if val != "None": + self.story.addToList('notes',val) + + if 'Contesto' in label: + self.story.setMetadata('context', value) + + ## Note--efp doesn't provide word count. + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?ssid=\d+&i=1")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1': + self.setSeries(series_name, i) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url)) + + div = soup.find('div', {'class' : 'storia'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/index.html b/index.html index e7d3d5c..31a3314 100644 --- a/index.html +++ b/index.html @@ -54,20 +54,11 @@ much easier.

-

New Sites:

+

New Site:

- Two new sites thanks to besnef: + New Italian language site:

-

-

New Fixes:

-

-

@@ -570,8 +561,12 @@ Use the URL of the story's first chapter, such as
http://www.jlaunlimited.com/eFiction1.1/viewstory.php?sid=1234 +

www.efpfanfic.net
+
+ Use the URL of any story chapter, such as +
http://www.efpfanfic.net/viewstory.php?sid=12345 +
-

A few additional things to know, which will make your life substantially easier: diff --git a/plugin-defaults.ini b/plugin-defaults.ini index 64e5c0e..c8b7b7f 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -896,6 +896,21 @@ extracategories:InuYasha extracharacters:Sesshoumaru,Kagome extraships:Sesshoumaru/Kagome +[www.efpfanfic.net] +## Some sites require login (or login for some rated stories) The +## program can prompt you, or you can save it in config. In +## commandline version, this should go in your personal.ini, not +## defaults.ini. +#username:YourName +#password:yourpassword + +## Extra metadata that this adapter knows about. See [dramione.org] +## for examples of how to use them. +extra_valid_entries:notes,context,type +notes_label:Notes +context_label:Context +type_label:Type of Couple + [www.fanfiction.net] ## fanfiction.net's 'cover' images are really just tiny thumbnails. ## Change this to false to use them anyway.