wassname/FanFicFare
1
0
Fork 0
mirror of https://github.com/wassname/FanFicFare.git synced 2026-09-11 11:52:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

New adapter for www.potterfics.com

Browse Source
This commit is contained in:
Dan
2013-01-02 07:20:05 +00:00
parent 3a76d65396
commit 9c245af0fd
4 changed files with 256 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
defaults.ini
+4
View File
@@ -1099,6 +1099,10 @@ extraships:Harry Potter/Ginny Weasley
## Site dedicated to these categories/characters/ships
extracategories:Harry Potter
[www.potterfics.com]
## Site dedicated to these categories/characters/ships
extracategories:Harry Potter
[www.prisonbreakfic.net]
## Site dedicated to these categories/characters/ships
extracategories:Prison Break
fanficdownloader/adapters/__init__.py
+1 -1
View File
@@ -107,7 +107,7 @@ import adapter_jlaunlimitedcom
import adapter_qafficcom
import adapter_efpfanficnet
#import adapter_buffynfaithnet
#import adapter_potterficscom
import adapter_potterficscom
## This bit of complexity allows adapters to be added by just adding
## importing. It eliminates the long if/else clauses we used to need
fanficdownloader/adapters/adapter_potterficscom.py
+247
View File
@@ -0,0 +1,247 @@
# -*- coding: utf-8 -*-
# Copyright 2013 Fanficdownloader team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import datetime
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from .. import BeautifulSoup as bs
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter
# This function is called by the downloader in all adapter_*.py files
# in this dir to register the adapter class. So it needs to be
# updated to reflect the class below it. That, plus getSiteDomain()
# take care of 'Registering'.
def getClass():
return PotterFicsComAdapter
# Class name has to be unique. Our convention is camel case the
# sitename with Adapter at the end. www is skipped.
class PotterFicsComAdapter(BaseSiteAdapter):
def __init__(self, config, url):
BaseSiteAdapter.__init__(self, config, url)
self.decode = ["Windows-1252",
"utf8"] # 1252 is a superset of iso-8859-1.
# Most sites that claim to be
# iso-8859-1 (and some that claim to be
# utf8) are really windows-1252.
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
self.password = ""
self.is_adult=False
# get storyId from url--url validation guarantees query correct
m = re.match(self.getSiteURLPattern(),url)
if m:
self.story.setMetadata('storyId',m.group('id'))
logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
# normalized story URL. gets rid of chapter if there, left with chapter index URL
nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
self._setURL(nurl)
else:
raise exceptions.InvalidStoryURL(url,
self.getSiteDomain(),
self.getSiteExampleURLs())
logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
# Each adapter needs to have a unique site abbreviation.
self.story.setMetadata('siteabbrev','potficscom')
@staticmethod # must be @staticmethod, don't remove it.
def getSiteDomain():
# The site domain. Does have www here, if it uses it.
return 'www.potterfics.com'
def getSiteExampleURLs(self):
return \
"http://www.potterfics.com/historias/127583 "\
"http://www.potterfics.com/historias/127583/capitulo-1 "\
"http://www.potterfics.com/historias/127583/capitulo-4 "\
"http://www.potterfics.com/historias/92810 "\
"http://www.potterfics.com/historias/111194"
def getSiteURLPattern(self):
#http://www.potterfics.com/historias/127583
#http://www.potterfics.com/historias/127583/capitulo-1
#http://www.potterfics.com/historias/127583/capitulo-4
#http://www.potterfics.com/historias/92810 -> Complete story
#http://www.potterfics.com/historias/111194 -> Complete, single chap
p = re.escape("http://"+self.getSiteDomain()+"/historias/")+\
r"(?P<id>\d+)(/capitulo-(?P<ch>\d+))?/?$"
return p
def extractChapterUrlsAndMetadata(self):
#this converts '/historias/12345' to 'http://www.potterfics.com/historias/12345'
def makeAbsoluteURL(url):
if url[0] == '/':
url = 'http://'+self.getSiteDomain()+url
return url
#use this to get month numbers from Spanish months
SpanishMonths = {
'enero' : '01',
'febrero' : '02',
'marzo' : '03',
'abril' : '04',
'mayo' : '05',
'junio' : '06',
'julio' : '07',
'agosto' : '08',
'septiembre' : '09',
'octubre' : '10',
'noviembre' : '11',
'diciembre' : '12'
}
url = self.url
logger.debug("URL: "+url)
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
raise e
#print data
#deal with adult content warnings - doesn't seem to apply to this site
#set constant meta for this site:
#Set Language = Spanish
self.story.setMetadata('language', 'Spanish')
#Set Category = Harry Potter
# This is better done in plugin-defaults.ini and defaults.ini
# by adding a section for this site with the line:
# extracategories:Harry Potter
#self.story.addToList('category','Harry Potter')
#get the rest of the meta
# use BeautifulSoup HTML parser to make everything easier to find.
#self closing br and img present!
soup = bs.BeautifulSoup(data,selfClosingTags=('br','img'))
#we want the second table directly under the body, contains all the metadata
table = soup.html.body.findAll('table', recursive=False)[1]
#within that, we want the second row, first cell
cell = table.tr.findNextSibling('tr').td
#find first metadata block
mb = cell.div.findNextSibling('div')
#Get meta...
self.story.setMetadata('title', mb.b.string)
#strip out brackets on rating
self.story.setMetadata('rating', mb.span.string[1:-1])
#Completion status is denoted by the presence of this image:
if mb.find('img',title="Historia terminada"):
self.story.setMetadata('status', 'Completed')
else:
self.story.setMetadata('status', 'In-Progress')
#find next metadata block
#author details
mb = mb.findNextSibling('div')
self.story.setMetadata('author', mb.b.a.string.strip())
self.story.setMetadata('authorUrl', makeAbsoluteURL(mb.b.a['href']))
self.story.setMetadata('authorId', self.story.getMetadata('authorUrl').split('/')[4])
#dates and times
mb = mb.find('span')
#posted/published = Escrita
date = mb.find(text=re.compile('Escrita el ')).strip().split()
year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
month = int(SpanishMonths[date[5].lower()])
day = int(date[3])
time = date[8].split(':')
hour = int(time[0])
minute = int(time[1])
self.story.setMetadata('datePublished', datetime.datetime(year, month, day, hour, minute))
#updated = Actualizada
date = mb.find(text=re.compile('Actualizada el ')).strip().split()
year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
month = int(SpanishMonths[date[5].lower()])
day = int(date[3])
time = date[8].split(':')
hour = int(time[0])
minute = int(time[1])
self.story.setMetadata('dateUpdated', datetime.datetime(year, month, day, hour, minute))
mb = mb.span.findNextSibling('span').findNextSibling('span')
wc = mb.find(text=re.compile(' palabras en total')).strip()
self.story.setMetadata('numWords', wc.split()[0])
#then we come to categories and genres. Oh dear. On this site, categories hold everything from genre, to ships, to crossovers.
#To make things worse, there is also another genre field, which often holds similar/duplicate info. Links to genre pages do not work
#though, so perhaps those will be phased out?
#for now, put them all into the genre list
links = mb.findAll('a',href=re.compile('/(categorias|generos)/\d+'))
genlist = [i.string.strip() for i in links]
self.story.extendList('genre',genlist)
#get the chapter urls
#we can go back to the table cell we found before
#get its last element and work backwards to find the last ordered list on the page
list = cell.contents[len(cell)-1].findPrevious('ol')
chapters = []
revs = 0
chnum = 0
for li in list:
chnum += 1
chTitle = str(chnum) + '. ' + li.a.b.string.strip()
chURL = makeAbsoluteURL(li.a['href'])
chapters.append((chTitle,chURL))
#Get reviews, add to total
revs += int(li.div.a.string.split()[0])
self.chapterUrls.extend(chapters)
self.story.setMetadata('numChapters', len(chapters))
self.story.setMetadata('reviews', revs)
#Now for the description... this may be tricky...
#if it is there (doesn't have to be), it will be before the chapter list,
#separated by a horizontal rule, and after the google ad bar
#get list's parent div
mb = list.parent
#get the div before that, will either be the description, or the google ad bar
mb = mb.findPreviousSibling('div')
if 'google_ad_client' in str(mb):
#couldn't find description, leaving it blank
pass
else:
self.setDescription(url,mb)
# grab the text for an individual chapter.
def getChapterText(self, url):
logger.debug('Getting chapter text from: %s' % url)
soup = bs.BeautifulSoup(self._fetchUrl(url),
selfClosingTags=('br','hr','img'))
div = soup.find('div', {'id' : 'cuerpoHistoria'})
if None == div:
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
return self.utf8FromSoup(url,div)
plugin-defaults.ini
+4
View File
@@ -1076,6 +1076,10 @@ extraships:Harry Potter/Ginny Weasley
## Site dedicated to these categories/characters/ships
extracategories:Harry Potter
[www.potterfics.com]
## Site dedicated to these categories/characters/ships
extracategories:Harry Potter
[www.prisonbreakfic.net]
## Site dedicated to these categories/characters/ships
extracategories:Prison Break