From 86f037543c46725ef7c269f4f6982e6e8735bcf5 Mon Sep 17 00:00:00 2001 From: Besnef Date: Sun, 30 Sep 2012 21:48:08 -0400 Subject: [PATCH] Actually add the adapter files for dwiggie.com, jlaunlimited.com & indeath.net(blog format) --- .../adapters/adapter_dwiggiecom.py | 287 ++++++++++++++++++ .../adapters/adapter_indeathnet.py | 203 +++++++++++++ .../adapters/adapter_jlaunlimitedcom.py | 283 +++++++++++++++++ 3 files changed, 773 insertions(+) create mode 100644 fanficdownloader/adapters/adapter_dwiggiecom.py create mode 100644 fanficdownloader/adapters/adapter_indeathnet.py create mode 100644 fanficdownloader/adapters/adapter_jlaunlimitedcom.py diff --git a/fanficdownloader/adapters/adapter_dwiggiecom.py b/fanficdownloader/adapters/adapter_dwiggiecom.py new file mode 100644 index 0000000..436157b --- /dev/null +++ b/fanficdownloader/adapters/adapter_dwiggiecom.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return DwiggieComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class DwiggieComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + +# # get storyId from url--url validation guarantees query is only sid=1234 +# self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) +# logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + + + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + # normalized story URL. + self._setURL('http://www.' + self.getSiteDomain() + '/derby/'+self.story.getMetadata('storyId')+'.htm') + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','dwg') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'dwiggie.com' + + @classmethod + def getAcceptDomains(cls): + return ['www.dwiggie.com','dwiggie.com'] + + def getSiteExampleURLs(self): + return "http://"+self.getSiteDomain()+"/derby/name1b.htm" + + def getSiteURLPattern(self): + # http://www.dwiggie.com/derby/mari17b.htm + return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"/derby/(?P[a-z]+\d+)(?P[a-z]*)\.htm$" + + def tryArchivePage(self, url): + + try: + data = self._fetchUrl(url) + + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.meta) # need to change the exception returned + else: + raise e + + archivesoup = bs.BeautifulSoup(data) + m = re.compile(r"/derby/"+self.story.getMetadata('storyId')+"[a-z]?.htm$") + #print m.pattern + #print archivesoup + a = archivesoup.find('a', href=m) #http://www.indeath.net/user/9083-cyrex/ + + return a + + def getGenre(self, url): + if re.search('id=E',url): + genre='Epilogue Abbey' + else: + genre='Fantasia Gallery' + self.story.addToList('genre',genre) + + def getItemFromArchivePage(self): + + urls = ["http://www.dwiggie.com/toc/index.php?id=E&page=all&comp=n","http://www.dwiggie.com/toc/index.php?id=F&page=all&comp=n"] + for url in urls: + a = self.tryArchivePage(url) + if a != None: + self.getGenre(url) + return a.parent + else: + return None + + + def getMetaFromSearch(self): + + params = {} + params['title_name'] = self.story.getMetadata('title') + + searchUrl = "http://" + self.getSiteDomain() + "/toc/search.php" + + d = self._postUrl(searchUrl, params) + #print d + + searchsoup = bs.BeautifulSoup(d) + m = re.compile(r"/derby/"+self.story.getMetadata('storyId')+"[a-z]?.htm$") + #print m.pattern + #print self.story.getMetadata('storyId') + a = searchsoup.find('a', href=m) #http://www.indeath.net/user/9083-cyrex/ + + return a + + + def getChaptersFromPage(self, url): + data = self._fetchUrl(url) + + m = re.match('.*?]*>(\s*
    )?(?P.*?)', data, re.DOTALL) + newdata = m.group('content') + regex=re.compile(r'Continued\ In\ Next\ Section') + newdata = re.sub(regex, '', newdata) + + pagesections = filter(lambda x:x!=None, re.split('(?m)|

    \s*\s*<\/p>', newdata, re.MULTILINE)) + pagesections = filter(lambda x:x.strip()!='/', pagesections) + pagesections.pop(0) # always remove header + + regex = re.compile(r'(href\="'+self.story.getMetadata('storyId')+'[a-z]?.htm\"|Copyright\ held\ by\ the\ author)') + pagesections = filter(lambda x: not regex.search(x), pagesections) + return pagesections + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + meta = self.getItemFromArchivePage() + print meta + + # Title + t = meta.a + self.story.setMetadata('title',t.string.strip()) + + # Author + author = meta.find('a','author_link') + if author != None: + self.story.setMetadata('author',author.string.strip()) + self.story.setMetadata('authorId',author['href'].split('=')[1]) + self.story.setMetadata('authorUrl',author['href']) + author=author.parent + else: + author=meta.i + self.story.setMetadata('author',author.string.replace('Written by','').strip()) + + + # DateUpdated + dUpdate = meta.find('i',text = re.compile('Last update')) + du = dUpdate.replace('Last update','').replace('.','').strip() + self.story.setMetadata('dateUpdated', makeDate(du, self.dateformat)) + compImg=meta.find('img',alt="Dot") + if compImg != None: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + + # Summary & Category + # Get the summary components from the meta listing + metalist=meta.contents + s=[] + for x in xrange(0,len(metalist)-1): + item=metalist[x] + if item==author or item==compImg: + s=[] + continue + if item==dUpdate or item==dUpdate.parent: + break + s.append(item) + + # create a soup object from the summary components + soup=bs.BeautifulSoup("

    ") + d=soup.p + for x in s: + d.append(x) + + # extract category from summary text + desc=stripHTML(d) + books = re.compile(r'(?P\~P&P;?\~|\~Em;?\~|\~MP;?\~|\~S\&S;?\~|\~Per;?\~|\~NA;?\~|\~Juv;?\~|\~Misc;?\~)') + m=re.search(books,desc) + book=m.group('book') + self.story.addToList('category',book.replace(';','')) + + # assign summary info + if desc != None: + self.setDescription(url,stripHTML(desc).replace(book,'').strip()) + + + + + + ## Chapters (Sections in this case - don't know if we can subdivide them) + + # get the last Section from the archive page link + #chapters = ["http://www.dwiggie.com"+t['href']] + + # get the section letter from the last page + m = re.match("/derby/"+self.story.getMetadata('storyId')+"(?P
    [a-z]?).htm$",t['href']) + inc = m.group('section') + + # get the presumed list of section urls with 'lower' section letters + sections = [] + baseurl = "http://www.dwiggie.com/derby/"+self.story.getMetadata('storyId') + extension = ".htm" + ordend = ord(inc) + ordbegin = ord('a') + for numinc in xrange(ordbegin,ordend+1): + inc = chr(numinc) + if inc == 'a': + sections.append(baseurl+extension) + else: + sections.append(baseurl+inc+extension) + + + + # Process List of Chapters + # create 'dummy' urls for individual chapters in the form 'pageurl#pageindex' where page index is an index starting with 0 per page + c = 0 + postdate=None + for x in range(0,len(sections)): + section=sections[x] + i=0 + for chapter in self.getChaptersFromPage(section): + c+=1 + #self.chapterUrls.append(('Chapter '+str(c),section+'#'+str(i))) + self.chapterUrls.append(('Chapter '+str(c),section+'#'+str(i))) + if postdate==None: + regex=re.compile(r'Posted\ on\ (?P\d{4}\-\d{2}\-\d{2})') + m=re.search(regex,chapter) + if m!=None: + postdate=m.group('date') + self.story.setMetadata('datePublished', makeDate(postdate, "%Y-%m-%d")) + i+=1 + + self.story.setMetadata('numChapters',c) + logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters')) + + + + # grab the text for an individual chapter. + def getChapterText(self, url): + logging.debug('Getting chapter text from: %s' % url) + + + page_url = url.split('#')[0] + x = url.split('#')[1] + chapter = bs.BeautifulSoup(self.getChaptersFromPage(page_url)[int(x)]) + + return self.utf8FromSoup(url,chapter) diff --git a/fanficdownloader/adapters/adapter_indeathnet.py b/fanficdownloader/adapters/adapter_indeathnet.py new file mode 100644 index 0000000..57ab6c3 --- /dev/null +++ b/fanficdownloader/adapters/adapter_indeathnet.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return InDeathNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class InDeathNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + +# # get storyId from url--url validation guarantees query is only sid=1234 +# self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) +# logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + + + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + # normalized story URL. + self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/') + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','idn') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %B %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'indeath.net' + + @classmethod + def getAcceptDomains(cls): + return ['www.indeath.net','indeath.net'] + + def getSiteExampleURLs(self): + return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/" + + def getSiteURLPattern(self): + # http://www.indeath.net/blog/archive/169-ransom-in-death/ + return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"/blog/(archive/)?(?P\d+)\-(?P[a-z0-9\-]*)/?$" + + ## Login + def needToLoginCheck(self, data): + if 'This work is only available to registered users of the Archive.' in data \ + or "The password or user name you entered doesn't match our records" in data: + return True + else: + return False + + def getDateFromComponents(self, postmonth, postday): + ym = re.search(re.compile(r"Entries\ in\ (?PJanuary|February|March|April|May|June|July|August|September|October|November|December)\ (?P\d{4})"),postmonth) + d = re.search(re.compile(r"(?P\d{2})\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)"),postday) + postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat) + return postdate + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + try: + data = self._fetchUrl(url) +# meta = self._fetchUrl(metaurl) + +# if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta: +# raise exceptions.AdultCheckRequired(self.url) + + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.meta) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url,data) + data = self._fetchUrl(url) + meta = self._fetchUrl(metaurl) + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + # Now go hunting for all the meta data and the chapter list. + + ## Title + h = soup.find('a', id="blog_title") + t = h.find('span') + self.story.setMetadata('title',t.contents[0].string.strip()) + + s = t.find('div') + if s != None: + self.setDescription(url,s) + #self.story.setMetadata('description',s.text) + + # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required) + a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$")) #http://www.indeath.net/user/9083-cyrex/ + m = re.search(re.compile(r'http://www.indeath.net/user/(?P\d+)\-(?P[a-z0-9]*)/$'),a['href']) + self.story.setMetadata('authorId',m.group('id')) + self.story.setMetadata('authorUrl',a['href']) + self.story.setMetadata('author',m.group('name')) + + + + + # Find the chapters: + chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$")) + + #reverse the list since newest at the top + chapters.reverse() + + # Get date published & updated from first & last entries + posttable=soup.find('div', id="main_column") + + postmonths=posttable.findAll('th', text=re.compile(r'Entries\ in\ ')) + postmonths.reverse() + + postdates=posttable.findAll('span', _class="desc", text=re.compile('\d{2}\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)')) + postdates.reverse() + + self.story.setMetadata('datePublished',self.getDateFromComponents(postmonths[0],postdates[0])) + self.story.setMetadata('dateUpdated',self.getDateFromComponents(postmonths[len(postmonths)-1],postdates[len(postdates)-1])) + + # Process List of Chapters + self.story.setMetadata('numChapters',len(chapters)) + logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters')) + for x in range(0,len(chapters)): + # just in case there's tags, like in chapter titles. + chapter=chapters[x] + if len(chapters)==1: + self.chapterUrls.append((self.story.getMetadata('title'),chapter['href'])) + else: + ct = stripHTML(chapter) + tnew = re.match(re.compile(r"(?i)"+self.story.getMetadata('title')+r" - (?P.*)$"),ct) + if tnew: + chaptertitle = tnew.group('newtitle') + else: + chaptertitle = ct + self.chapterUrls.append((chaptertitle,chapter['href'])) + + + + + # grab the text for an individual chapter. + def getChapterText(self, url): + logging.debug('Getting chapter text from: %s' % url) + + #chapter=bs.BeautifulSoup('
    ') + data = self._fetchUrl(url) + soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center')) + + if None == soup: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + #print data + #chapter = soup.find('div', {'class' : "entry_content"}) + chapter = soup.find("div", "entry_content") + + return self.utf8FromSoup(url,chapter) diff --git a/fanficdownloader/adapters/adapter_jlaunlimitedcom.py b/fanficdownloader/adapters/adapter_jlaunlimitedcom.py new file mode 100644 index 0000000..5ffb4f4 --- /dev/null +++ b/fanficdownloader/adapters/adapter_jlaunlimitedcom.py @@ -0,0 +1,283 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + + +def getClass(): + return JLAUnlimitedComAdapter + + +class JLAUnlimitedComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + + self._setURL('http://' + self.getSiteDomain() + '/eFiction1.1/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','jla') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.jlaunlimited.com' + + def getSiteExampleURLs(self): + return "http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. This story is in The Bedchamber + def needToLoginCheck(self, data): + if 'This story is in The Bedchamber' in data \ + or 'That username is not in our database' in data \ + or "That password is not correct, please try again" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['name'] = self.username + params['pass'] = self.password + else: + params['name'] = self.getConfig("username") + params['pass'] = self.getConfig("password") + params['login'] = 'yes' + params['submit'] = 'login' + + loginUrl = 'http://' + self.getSiteDomain()+'/login.php' + d = self._fetchUrl(loginUrl,params) + e = self._fetchUrl(url) + + if "Welcome back," not in d : #Member Account + logging.info("Failed to login to URL %s as %s" % (loginUrl, + params['name'])) + raise exceptions.FailedToLogin(url,params['name']) + return False + elif "This story is in The Bedchamber" in e: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Your account does not have sufficient priviliges to read this story.") + return False + else: + return True + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logging.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if "Age Consent Required" in data: # XXX + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId'))) + self.story.setMetadata('title',a.string) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eFiction1.1/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + print len(self.chapterUrls) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Rated: NC-17
    etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + ## Not all sites use Genre, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/fanfic/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logging.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div)