diff --git a/app.yaml b/app.yaml index f6428d3..852b064 100644 --- a/app.yaml +++ b/app.yaml @@ -1,20 +1,22 @@ -# fanfictionloader ffd-retief -application: fanfictionloader -version: 4-0-7 -runtime: python +# ffd-retief-hrd fanfictiondownloader +application: fanfictiondownloader +version: 4-1-1 +runtime: python27 api_version: 1 +threadsafe: true handlers: + - url: /r3m0v3r.* - script: utils/remover.py + script: utils.remover.app login: admin - url: /tally.* - script: utils/tally.py + script: utils.tally.app login: admin - url: /fdownloadtask - script: main.py + script: main.app login: admin - url: /css @@ -31,7 +33,14 @@ handlers: upload: static/favicon\.ico - url: /.* - script: main.py + script: main.app builtins: - datastore_admin: on + +libraries: +- name: django + version: "1.2" + +- name: PIL + version: "1.1.7" diff --git a/cron.yaml b/cron.yaml index c097e5c..e72999f 100644 --- a/cron.yaml +++ b/cron.yaml @@ -3,6 +3,8 @@ cron: url: /r3m0v3r schedule: every 2 hours -- description: orphan cleanup job - url: /r3m0v3rOrphans - schedule: every 4 hours +# There's a bug in the Python 2.7 runtime that prevents this from +# working properly. In theory, there should never be orphans anyway. +#- description: orphan cleanup job +# url: /r3m0v3rOrphans +# schedule: every 4 hours diff --git a/defaults.ini b/defaults.ini index d591fb0..6da575a 100644 --- a/defaults.ini +++ b/defaults.ini @@ -113,6 +113,15 @@ extratags: FanFiction ## Primarily for commandline. #slow_down_sleep_time:0.5 +## output background color--only used by html and epub (and ignored in +## epub by many readers). Must be hex code, # will be added. +background_color: ffffff + +## For use only with stand-alone CLI version--run a command on the +## generated file after it's produced. All of the titlepage_entries +## values are available, plus output_filename. +#post_process_cmd: addbook -f "${output_filename}" -t "${title}" + ## Each output format has a section that overrides [defaults] [html] @@ -245,6 +254,12 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext} ## this should go in your personal.ini, not defaults.ini. #is_adult:true +[www.tthfanfic.org] +## Some sites do not require a login, but do require the user to +## confirm they are adult for adult content. In commandline version, +## this should go in your personal.ini, not defaults.ini. +#is_adult:true + [overrides] ## It may sometimes be useful to override all of the specific format, ## site and site:format sections in your private configuration. For diff --git a/downloader.py b/downloader.py index 1615178..f0b7065 100644 --- a/downloader.py +++ b/downloader.py @@ -24,6 +24,9 @@ from os.path import normpath, expanduser, isfile, join from StringIO import StringIO from optparse import OptionParser import getpass +import string + +from subprocess import call from epubmerge import doMerge @@ -38,7 +41,9 @@ import ConfigParser def writeStory(config,adapter,writeformat,metaonly=False,outstream=None): writer = writers.getWriter(writeformat,config,adapter) writer.writeStory(outstream=outstream,metaonly=metaonly) + output_filename=writer.getOutputFileName() del writer + return output_filename def main(): @@ -65,7 +70,7 @@ def main(): help="Update an existing epub with new chapter, give epub filename instead of storyurl. Not compatible with inserted TOC.",) parser.add_option("--force", action="store_true", dest="force", - help="Force update of an existing epub, download and overwrite all chapters.",) + help="Force overwrite or update of an existing epub, download and overwrite all chapters.",) (options, args) = parser.parse_args() @@ -97,6 +102,10 @@ def main(): config.add_section("overrides") except ConfigParser.DuplicateSectionError: pass + + if options.force: + config.set("overrides","always_overwrite","true") + if options.options: for opt in options.options: (var,val) = opt.split('=') @@ -112,7 +121,7 @@ def main(): striptitletoc=True, forceunique=False) print "Updating %s, URL: %s" % (args[0],url) - filename = args[0] + output_filename = args[0] config.set("overrides","output_filename",args[0]) else: url = args[0] @@ -184,16 +193,14 @@ def main(): adapter.setChaptersRange(options.begin,options.end) - if options.format == "all": - ## For testing. Doing all three formats actually causes - ## some interesting config issues with format-specific - ## sections. But it should rarely be an issue. - writeStory(config,adapter,"epub",options.metaonly) - writeStory(config,adapter,"html",options.metaonly) - writeStory(config,adapter,"txt",options.metaonly) - else: - writeStory(config,adapter,options.format,options.metaonly) + output_filename=writeStory(config,adapter,options.format,options.metaonly) + if not options.metaonly and adapter.getConfig("post_process_cmd"): + metadata = adapter.story.metadata + metadata['output_filename']=output_filename + call(string.Template(adapter.getConfig("post_process_cmd")) + .substitute(metadata)) + del adapter except exceptions.InvalidStoryURL, isu: diff --git a/epubmerge.py b/epubmerge.py index a9c5193..99e2523 100644 --- a/epubmerge.py +++ b/epubmerge.py @@ -216,7 +216,7 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None, try: outputepub.writestr(href, epub.read(relpath+item.getAttribute("href"))) - if re.match(r'.*/(file|chapter)\d+\.xhtml',href): + if re.match(r'.*/(file|chapter)\d+\.x?html',href): filecount+=1 items.append((id,href,item.getAttribute("media-type"))) filelist.append(href) diff --git a/fanficdownloader/adapters/adapter_fimfictionnet.py b/fanficdownloader/adapters/adapter_fimfictionnet.py index 0739184..8332203 100644 --- a/fanficdownloader/adapters/adapter_fimfictionnet.py +++ b/fanficdownloader/adapters/adapter_fimfictionnet.py @@ -88,7 +88,8 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"}) - title, author = [link.text for link in soup.find("h2").findAll("a")] + title = soup.find("h2").find("a").text # first a link in first h2 is title. + author = soup.find("h2").find("span",{'class':'author'}).find("a").text self.story.setMetadata("title", title) self.story.setMetadata("author", author) self.story.setMetadata("authorId", author) # The author's name will be unique @@ -158,4 +159,4 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): if soup == None: raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return utf8FromSoup(soup) - \ No newline at end of file + diff --git a/fanficdownloader/adapters/adapter_tthfanficorg.py b/fanficdownloader/adapters/adapter_tthfanficorg.py new file mode 100644 index 0000000..1906aaa --- /dev/null +++ b/fanficdownloader/adapters/adapter_tthfanficorg.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +import re +import urllib2 +import time + +import fanficdownloader.BeautifulSoup as bs +from fanficdownloader.htmlcleanup import stripHTML +import fanficdownloader.exceptions as exceptions + +from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate + +class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','tth') + self.dateformat = "%d %b %y" + self.is_adult=False + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + # normalized story URL. + self._setURL("http://"+self.getSiteDomain()\ + +"/Story-"+self.story.getMetadata('storyId')) + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + @staticmethod + def getSiteDomain(): + return 'www.tthfanfic.org' + + def getSiteExampleURLs(self): + return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm ttp://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm" + + # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm + # http://www.tthfanfic.org/Story-5583 + # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm + def getSiteURLPattern(self): + return r"http://www.tthfanfic.org/(T-\d+/)?Story-(?P\d+)(-\d+)?(/.*)?$" + + def extractChapterUrlsAndMetadata(self): + # fetch the chapter. From that we will get almost all the + # metadata and chapter list + + url=self.url + logging.debug("URL: "+url) + + # use BeautifulSoup HTML parser to make everything easier to find. + try: + data = self._fetchUrl(url) + soup = bs.BeautifulSoup(data) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(url) + else: + raise e + + if "

Story Not Found

" in data: + raise exceptions.StoryDoesNotExist(url) + + if "NOTE: This story is rated FR21 which is above your chosen filter level." in data: + if self.is_adult or self.getConfig("is_adult"): + form = soup.find('form', {'id':'sitemaxratingform'}) + params={'ctkn':form.find('input', {'name':'ctkn'})['value'], + 'sitemaxrating':'5'} + logging.info("Attempting to get rating cookie for %s" % url) + data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params) + # refetch story page. + data = self._fetchUrl(url) + soup = bs.BeautifulSoup(data) + else: + raise exceptions.AdultCheckRequired(self.url) + + # http://www.tthfanfic.org/AuthorStories-3449/Greywizard.htm + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"^/AuthorStories-\d+")) + self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+a['href']) + self.story.setMetadata('author',stripHTML(a)) + + try: + # going to pull part of the meta data from author list page. + logging.debug("author URL: "+self.story.getMetadata('authorUrl')) + authordata = self._fetchUrl(self.story.getMetadata('authorUrl')) + authorsoup = bs.BeautifulSoup(authordata) + # author can have several pages, scan until we find it. + while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ): + nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href'] + logging.debug("author nextpage URL: "+nextpage) + authordata = self._fetchUrl(nextpage) + authorsoup = bs.BeautifulSoup(authordata) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(url) + else: + raise e + + storydiv = authorsoup.find('div', {'id':'st'+self.story.getMetadata('storyId'), 'class':re.compile(r"storylistitem")}) + self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'}))) + self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'}))) + + verticaltable = soup.find('table', {'class':'verticaltable'}) + + BtVS = True + for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")): + if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover']: + self.story.addToList('category',cat.string) + else: + if cat.string == 'Non-BtVS/AtS Stories': + BtVS = False + if BtVS: + self.story.addToList('category','Buffy: The Vampire Slayer') + + verticaltabletds = verticaltable.findAll('td') + self.story.setMetadata('rating', verticaltabletds[2].string) + self.story.setMetadata('numWords', verticaltabletds[4].string) + + # Complete--if completed. + if 'Yes' in verticaltabletds[10].string: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + self.story.setMetadata('datePublished',makeDate(stripHTML(verticaltabletds[8].string), self.dateformat)) + self.story.setMetadata('dateUpdated',makeDate(stripHTML(verticaltabletds[9].string), self.dateformat)) + + for icon in storydiv.find('span',{'class':'storyicons'}).findAll('img'): + if( icon['title'] not in ['Non-Crossover'] ) : + self.story.addToList('genre',icon['title']) + + # Find the chapter selector + select = soup.find('select', { 'name' : 'chapnav' } ) + + if select is None: + # no selector found, so it's a one-chapter story. + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + allOptions = select.findAll('option') + for o in allOptions: + url = "http://"+self.host+o['value'] + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(o),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + return + + + def getChapterText(self, url): + logging.debug('Getting chapter text from: %s' % url) + soup = bs.BeautifulSoup(self._fetchUrl(url)) + + div = soup.find('div', {'id' : 'storyinnerbody'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + # strip out included chapter title, if present, to avoid doubling up. + try: + div.find('h3').extract() + except: + pass + return utf8FromSoup(div) + +def getClass(): + return TwistingTheHellmouthSiteAdapter + diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index ca48cde..68b24be 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -25,7 +25,7 @@ class Story: try: self.metadata = {'version':os.environ['CURRENT_VERSION_ID']} except: - self.metadata = {'version':'4.0'} + self.metadata = {'version':'4.1'} self.chapters = [] # chapters will be tuples of (title,html) self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists. diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py index ec9b11d..7b3fac2 100644 --- a/fanficdownloader/writers/writer_epub.py +++ b/fanficdownloader/writers/writer_epub.py @@ -41,7 +41,10 @@ class EpubWriter(BaseStoryWriter): def __init__(self, config, story): BaseStoryWriter.__init__(self, config, story) - self.EPUB_CSS='''body { margin-left: 2%; margin-right: 2%; margin-top: 2%; margin-bottom: 2%; text-align: justify; } + self.EPUB_CSS = string.Template(''' +body { margin: 2%; + text-align: justify; + background-color: #${background_color}; } pre { font-size: x-small; } sml { font-size: small; } h1 { text-align: center; } @@ -63,7 +66,7 @@ h6 { text-align: center; } .smcap {font-variant: small-caps;} .u {text-decoration: underline;} .bold {font-weight: bold;} -''' +''') self.EPUB_TITLE_PAGE_START = string.Template(''' @@ -376,7 +379,7 @@ h6 { text-align: center; } del tocncxdom # write stylesheet.css file. - outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS) + outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")})) # write title page. if self.getConfig("titlepage_use_table"): diff --git a/fanficdownloader/writers/writer_html.py b/fanficdownloader/writers/writer_html.py index 32d27d6..cec520a 100644 --- a/fanficdownloader/writers/writer_html.py +++ b/fanficdownloader/writers/writer_html.py @@ -39,6 +39,7 @@ class HTMLWriter(BaseStoryWriter): ${title} by ${author}