From 0cd71263b1d06cd1d721c0e18462f7aea4eba726 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Sun, 1 May 2016 21:54:55 -0500 Subject: [PATCH] Change adapter_ncisfictionnet to eFiction Base due to site change. --- fanficfare/adapters/adapter_ncisfictionnet.py | 201 ++---------------- 1 file changed, 14 insertions(+), 187 deletions(-) diff --git a/fanficfare/adapters/adapter_ncisfictionnet.py b/fanficfare/adapters/adapter_ncisfictionnet.py index ca59cd9..fa7d643 100644 --- a/fanficfare/adapters/adapter_ncisfictionnet.py +++ b/fanficfare/adapters/adapter_ncisfictionnet.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2012 Fanficdownloader team, 2015 FanFicFare team +# Copyright 2012 Fanficdownloader team, 2016 FanFicFare team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,195 +15,22 @@ # limitations under the License. # -import time -import logging -logger = logging.getLogger(__name__) -import re -import urllib2 +# Software: eFiction +from base_efiction_adapter import BaseEfictionAdapter +class NCISFictionNetAdapter(BaseEfictionAdapter): -from ..htmlcleanup import stripHTML -from .. import exceptions as exceptions + @staticmethod + def getSiteDomain(): + return 'ncisfiction.net' -from base_adapter import BaseSiteAdapter, makeDate + @classmethod + def getSiteAbbrev(self): + return 'ncisfn' + + @classmethod + def getDateFormat(self): + return "%m/%d/%Y" def getClass(): return NCISFictionNetAdapter - -# Class name has to be unique. Our convention is camel case the -# sitename with Adapter at the end. www is skipped. -class NCISFictionNetAdapter(BaseSiteAdapter): - - def __init__(self, config, url): - BaseSiteAdapter.__init__(self, config, url) - - self.decode = ["iso-8859-1", - "Windows-1252"] # 1252 is a superset of iso-8859-1. - # Most sites that claim to be - # iso-8859-1 (and some that claim to be - # utf8) are really windows-1252. - self.username = "NoneGiven" # if left empty, site doesn't return any message at all. - self.password = "" - self.is_adult=False - - # get storyId from url--url validation guarantees query is only sid=1234 - self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) - - - # normalized story URL. - self._setURL("http://"+self.getSiteDomain()\ - +"/chapters.php?stid="+self.story.getMetadata('storyId')) - - # Each adapter needs to have a unique site abbreviation. - self.story.setMetadata('siteabbrev','ncisfn') - - # The date format will vary from site to site. - # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - self.dateformat = "%d/%m/%Y" - - @staticmethod # must be @staticmethod, don't remove it. - def getSiteDomain(): - # The site domain. Does have www here, if it uses it. - return 'www.ncisfiction.net' - - ## Changed from www.ncisfiction.com to www.ncisfiction.net Oct - ## 2012 due to the ncisfiction.com domain expiring. Still accept - ## .com domains for existing updates, etc. - - @classmethod - def getAcceptDomains(cls): - return ['www.ncisfiction.net','www.ncisfiction.com'] - - @classmethod - def getSiteExampleURLs(cls): - return "http://"+cls.getSiteDomain()+"/story.php?stid=01234 http://"+cls.getSiteDomain()+"/chapters.php?stid=1234" - - def getSiteURLPattern(self): - return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+' - - - ## Getting the chapter list and the meta data, plus 'is adult' checking. - def extractChapterUrlsAndMetadata(self): - - # index=1 makes sure we see the story chapter index. Some - # sites skip that for one-chapter stories. - url = self.url - logger.debug("URL: "+url) - - try: - data = self._fetchUrl(url) - except urllib2.HTTPError, e: - if e.code == 404: - raise exceptions.StoryDoesNotExist(self.url) - else: - raise e - - - if "Access denied. This story has not been validated by the adminstrators of this site." in data: - raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") - - # use BeautifulSoup HTML parser to make everything easier to find. - soup = self.make_soup(data) - # print data - - # Now go hunting for all the meta data and the chapter list. - - ## Title and author - a = soup.find('div', {'class' : 'main_title'}) - - aut = a.find('a') - self.story.setMetadata('authorId',aut['href'].split('=')[1]) - self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href']) - self.story.setMetadata('author',aut.string) - - aut.extract() - self.story.setMetadata('title',stripHTML(a)[:len(stripHTML(a))-2]) - - # Find the chapters: - i=0 - chapters=soup.findAll('table', {'class' : 'story_table'}) - for chapter in chapters: - ch=chapter.find('a') - # just in case there's tags, like in chapter titles. - self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/'+ch['href'])) - if i == 0: - self.story.setMetadata('datePublished', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat)) - if i == len(chapters)-1: - self.story.setMetadata('dateUpdated', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat)) - i=i+1 - - self.story.setMetadata('numChapters',len(self.chapterUrls)) - - # eFiction sites don't help us out a lot with their meta data - # formating, so it's a little ugly. - - info = soup.find('table', {'class' : 'story_info'}) - - # no convenient way to calculate word count as it is logged differently for stories with and without series - - labels = info.findAll('tr') - for tr in labels: - value = tr.find('td') - label = tr.find('th').string - - if 'Summary' in label: - self.setDescription(url,value) - - if 'Rating' in label: - self.story.setMetadata('rating', value.string) - - if 'Category' in label: - cats = value.findAll('a') - for cat in cats: - self.story.addToList('category',cat.string) - - if 'Characters' in label: - chars = value.findAll('a') - for char in chars: - self.story.addToList('characters',char.string) - - if 'Pairing' in label: - ships = value.findAll('a') - for ship in ships: - self.story.addToList('ships',ship.string) - - if 'Genre' in label: - genres = value.findAll('a') - for genre in genres: - self.story.addToList('genre',genre.string) - - if 'Warnings' in label: - warnings = value.findAll('a') - for warning in warnings: - self.story.addToList('warnings',warning.string) - - if 'Status' in label: - if 'not completed' in value.text: - self.story.setMetadata('status', 'In-Progress') - else: - self.story.setMetadata('status', 'Completed') - - try: - # Find Series name from series URL. - a = soup.find('div',{'class' : 'sub_header'}) - series_name = a.find('a').string - i = a.text.split('#')[1] - self.setSeries(series_name, i) - self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href']) - except: - # I find it hard to care if the series parsing fails - pass - - # grab the text for an individual chapter. - def getChapterText(self, url): - - logger.debug('Getting chapter text from: %s' % url) - - soup = self.make_soup(self._fetchUrl(url)) - - div = soup.find('div', {'class' : 'story_text'}) - - if None == div: - raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - - return self.utf8FromSoup(url,div)