mirror of
https://github.com/wassname/FanFicFare.git
synced 2026-09-15 11:36:30 +08:00
Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Jim Miller
|
7eb890e231 | ||
|
Jim Miller
|
eeb4204797 | ||
|
Jim Miller
|
4f63c646b7 | ||
|
Ida
|
a91aac8d2d | ||
|
Ida
|
7f50b87842 | ||
|
Jim Miller
|
333c369b0d | ||
|
Ida
|
dd5817540b | ||
|
Ida
|
609ebe71b8 | ||
|
Jim Miller
|
b34b8fa9db | ||
|
Jim Miller
|
9e74ccb109 |
@@ -1,6 +1,6 @@
|
||||
# ffd-retief-hrd fanfictiondownloader
|
||||
application: fanfictiondownloader
|
||||
version: 4-4-20
|
||||
version: 4-4-21
|
||||
runtime: python27
|
||||
api_version: 1
|
||||
threadsafe: true
|
||||
|
||||
@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
|
||||
description = 'UI plugin to download FanFiction stories from various sites.'
|
||||
supported_platforms = ['windows', 'osx', 'linux']
|
||||
author = 'Jim Miller'
|
||||
version = (1, 5, 44)
|
||||
version = (1, 5, 45)
|
||||
minimum_calibre_version = (0, 8, 30)
|
||||
|
||||
#: This field defines the GUI plugin class that contains all the code
|
||||
|
||||
@@ -497,6 +497,7 @@ make_firstimage_cover:true
|
||||
book['password'] = adapter.password
|
||||
|
||||
book['icon'] = 'plus.png'
|
||||
book['status'] = 'Add'
|
||||
if story.getMetadataRaw('datePublished'):
|
||||
# should only happen when an adapter is broken, but better to
|
||||
# fail gracefully.
|
||||
@@ -505,6 +506,7 @@ make_firstimage_cover:true
|
||||
|
||||
if collision in (CALIBREONLY):
|
||||
book['icon'] = 'metadata.png'
|
||||
book['status'] = 'Meta'
|
||||
|
||||
# Dialogs should prevent this case now.
|
||||
if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
|
||||
@@ -560,6 +562,7 @@ make_firstimage_cover:true
|
||||
book_id = identicalbooks.pop()
|
||||
book['calibre_id'] = book_id
|
||||
book['icon'] = 'edit-redo.png'
|
||||
book['status'] = 'Update'
|
||||
|
||||
if book_id != None and collision != ADDNEW:
|
||||
if collision in (CALIBREONLY):
|
||||
@@ -767,47 +770,89 @@ make_firstimage_cover:true
|
||||
|
||||
self.previous = self.gui.library_view.currentIndex()
|
||||
db = self.gui.current_db
|
||||
# if display_story_list(self.gui,
|
||||
# 'Downloads finished, confirm to update Calibre',
|
||||
# prefs,
|
||||
# self.qaction.icon(),
|
||||
# job.result,
|
||||
# label_text='Stories will not be added or updated in Calibre without confirmation.',
|
||||
# offer_skip=True):
|
||||
|
||||
if display_story_list(self.gui,
|
||||
'Downloads finished, confirm to update Calibre',
|
||||
prefs,
|
||||
self.qaction.icon(),
|
||||
job.result,
|
||||
label_text='Stories will not be added or updated in Calibre without confirmation.',
|
||||
offer_skip=True):
|
||||
|
||||
book_list = job.result
|
||||
good_list = filter(lambda x : x['good'], book_list)
|
||||
total_good = len(good_list)
|
||||
# payload = (job.statistics_cols_map, book_statistics_map)
|
||||
# all_ids = set(book_statistics_map.keys())
|
||||
# msg = '<p>Count Pages plugin found <b>%d statistics(s)</b>. ' % len(all_ids) + \
|
||||
# 'Proceed with updating columns in your library?'
|
||||
# self.gui.proceed_question(self._update_database_columns,
|
||||
# payload, job.details,
|
||||
# 'Count log', 'Count complete', msg,
|
||||
# show_copy_button=False)
|
||||
|
||||
self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
|
||||
book_list = job.result
|
||||
good_list = filter(lambda x : x['good'], book_list)
|
||||
bad_list = filter(lambda x : x['calibre_id'] and not x['good'], book_list)
|
||||
payload = (good_list, bad_list, options)
|
||||
|
||||
msg = '''
|
||||
<p>FFDL found <b>%s</b> good and <b>%s</b> bad updates.</p>
|
||||
<p>See log for details.</p>
|
||||
<p>Proceed with updating your library?</p>
|
||||
'''%(len(good_list),len(bad_list))
|
||||
|
||||
if total_good > 0:
|
||||
htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>URL</th><th>Comment</th></tr>'
|
||||
for book in good_list:
|
||||
if 'status' in book:
|
||||
status = book['status']
|
||||
else:
|
||||
status = 'Good'
|
||||
htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
|
||||
|
||||
for book in bad_list:
|
||||
if 'status' in book:
|
||||
status = book['status']
|
||||
else:
|
||||
status = 'Bad'
|
||||
htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
|
||||
|
||||
htmllog = htmllog + '</table></body></html>'
|
||||
|
||||
self.gui.proceed_question(self._do_download_list_update,
|
||||
payload, htmllog,
|
||||
'FFDL log', 'FFDL download complete', msg,
|
||||
show_copy_button=True)
|
||||
|
||||
|
||||
def _do_download_list_update(self, payload):
|
||||
|
||||
(good_list,bad_list,options) = payload
|
||||
total_good = len(good_list)
|
||||
|
||||
self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
|
||||
|
||||
if total_good > 0:
|
||||
LoopProgressDialog(self.gui,
|
||||
good_list,
|
||||
partial(self._update_book, options=options, db=self.gui.current_db),
|
||||
partial(self._update_books_completed, options=options),
|
||||
init_label="Updating calibre for FanFiction stories...",
|
||||
win_title="Update calibre for FanFiction stories",
|
||||
status_prefix="Updated")
|
||||
|
||||
total_bad = len(bad_list)
|
||||
|
||||
if total_bad > 0:
|
||||
custom_columns = self.gui.library_view.model().custom_columns
|
||||
if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
|
||||
self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%total_bad))
|
||||
label = custom_columns[prefs['errorcol']]['label']
|
||||
## if error column and all bad.
|
||||
LoopProgressDialog(self.gui,
|
||||
good_list,
|
||||
partial(self._update_book, options=options, db=self.gui.current_db),
|
||||
partial(self._update_books_completed, options=options),
|
||||
init_label="Updating calibre for FanFiction stories...",
|
||||
win_title="Update calibre for FanFiction stories",
|
||||
bad_list,
|
||||
partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
|
||||
partial(self._update_books_completed, options=options, showlist=False),
|
||||
init_label="Updating calibre for BAD FanFiction stories...",
|
||||
win_title="Update calibre for BAD FanFiction stories",
|
||||
status_prefix="Updated")
|
||||
|
||||
bad_list = filter(lambda x : x['calibre_id'] and not x['good'], book_list)
|
||||
total_bad = len(bad_list)
|
||||
|
||||
if total_bad > 0:
|
||||
custom_columns = self.gui.library_view.model().custom_columns
|
||||
if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
|
||||
self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%total_bad))
|
||||
label = custom_columns[prefs['errorcol']]['label']
|
||||
## if error column and all bad.
|
||||
LoopProgressDialog(self.gui,
|
||||
bad_list,
|
||||
partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
|
||||
partial(self._update_books_completed, options=options, showlist=False),
|
||||
init_label="Updating calibre for BAD FanFiction stories...",
|
||||
win_title="Update calibre for BAD FanFiction stories",
|
||||
status_prefix="Updated")
|
||||
|
||||
|
||||
def _add_or_update_book(self,book,options,prefs,mi=None):
|
||||
db = self.gui.current_db
|
||||
@@ -830,6 +875,7 @@ make_firstimage_cover:true
|
||||
book['comment'] = "Adding format to book failed for some reason..."
|
||||
book['good']=False
|
||||
book['icon']='dialog_error.png'
|
||||
book['status'] = 'Error'
|
||||
|
||||
if prefs['deleteotherforms']:
|
||||
fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
|
||||
@@ -1117,6 +1163,7 @@ make_firstimage_cover:true
|
||||
book['comment'] = "No story URL found."
|
||||
book['good'] = False
|
||||
book['icon'] = 'search_delete_saved.png'
|
||||
book['status'] = 'Not Found'
|
||||
else:
|
||||
# get normalized url or None.
|
||||
book['url'] = self._is_good_downloader_url(url)
|
||||
@@ -1125,6 +1172,7 @@ make_firstimage_cover:true
|
||||
book['comment'] = "URL is not a valid story URL."
|
||||
book['good'] = False
|
||||
book['icon']='dialog_error.png'
|
||||
book['status'] = 'Bad URL'
|
||||
|
||||
def _get_story_url(self, db, book_id):
|
||||
identifiers = db.get_identifiers(book_id,index_is_id=True)
|
||||
|
||||
@@ -177,6 +177,7 @@ def do_download_for_worker(book,options):
|
||||
book['good']=False
|
||||
book['comment']=unicode(e)
|
||||
book['icon']='dialog_error.png'
|
||||
book['status'] = 'Error'
|
||||
print("Exception: %s:%s"%(book,unicode(e)))
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
@@ -423,6 +423,21 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
[grangerenchanted.com]
|
||||
## Some sites require login (or login for some rated stories) The
|
||||
## program can prompt you, or you can save it in config. In
|
||||
## commandline version, this should go in your personal.ini, not
|
||||
## defaults.ini.
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
## Some sites also require the user to confirm they are adult for
|
||||
## adult content. In commandline version, this should go in your
|
||||
## personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
[hlfiction.net]
|
||||
|
||||
[lumos.sycophanthex.com]
|
||||
## Some sites do not require a login, but do require the user to
|
||||
## confirm they are adult for adult content. In commandline version,
|
||||
@@ -439,6 +454,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
|
||||
## this should go in your personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
[nha.magical-worlds.us]
|
||||
|
||||
[occlumency.sycophanthex.com]
|
||||
## Some sites require login (or login for some rated stories) The
|
||||
## program can prompt you, or you can save it in config. In
|
||||
|
||||
@@ -67,10 +67,10 @@ class FineStoriesComAdapter(BaseSiteAdapter):
|
||||
return 'finestories.com'
|
||||
|
||||
def getSiteExampleURLs(self):
|
||||
return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/s/toryInfo.php?id=10537"
|
||||
return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/s/")+r"(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
|
||||
return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
@@ -160,13 +160,22 @@ class FineStoriesComAdapter(BaseSiteAdapter):
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# surprisingly, the detailed page does not give enough details, so go to author's page
|
||||
asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
|
||||
|
||||
for lc2 in asoup.findAll('td', {'class' : 'lc2'}):
|
||||
if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
|
||||
break
|
||||
skip=0
|
||||
i=0
|
||||
while i == 0:
|
||||
asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
|
||||
|
||||
self.story.addToList('category',lc2.find('div', {'class' : 'typediv'}).text)
|
||||
a = asoup.findAll('td', {'class' : 'lc2'})
|
||||
for lc2 in a:
|
||||
if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
|
||||
i=1
|
||||
break
|
||||
if a[len(a)-1] == lc2:
|
||||
skip=skip+10
|
||||
|
||||
for cat in lc2.findAll('div', {'class' : 'typediv'}):
|
||||
self.story.addToList('category',cat.text)
|
||||
|
||||
self.story.setMetadata('numWords', lc2.findNext('td', {'class' : 'num'}).text)
|
||||
|
||||
@@ -202,14 +211,14 @@ class FineStoriesComAdapter(BaseSiteAdapter):
|
||||
self.story.addToList('genre',genre)
|
||||
|
||||
if 'Posted' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
|
||||
|
||||
if 'Concluded' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
|
||||
|
||||
status = lc4.find('span', {'class' : 'ab'})
|
||||
if status != None:
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import time
|
||||
import logging
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
from .. import BeautifulSoup as bs
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return GrangerEnchantedCom
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class GrangerEnchantedCom(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
|
||||
self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
|
||||
|
||||
# normalized story URL.
|
||||
if "malfoymanor" in self.parsedUrl.netloc:
|
||||
self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
self.story.addToList("category","The Manor")
|
||||
else:
|
||||
self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','gech')
|
||||
|
||||
# If all stories from the site fall into the same category,
|
||||
# the site itself isn't likely to label them as such, so we
|
||||
# do.
|
||||
self.story.addToList("category","Harry Potter")
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%d/%b/%Y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'grangerenchanted.com'
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
|
||||
|
||||
def getSiteExampleURLs(self):
|
||||
return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$"
|
||||
|
||||
## Login seems to be reasonably standard across eFiction sites.
|
||||
def needToLoginCheck(self, data):
|
||||
if 'Registered Users Only' in data \
|
||||
or 'There is no such account on our website' in data \
|
||||
or "That password doesn't match the one in our database" in data:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def performLogin(self, url):
|
||||
params = {}
|
||||
|
||||
if self.password:
|
||||
params['penname'] = self.username
|
||||
params['password'] = self.password
|
||||
else:
|
||||
params['penname'] = self.getConfig("username")
|
||||
params['password'] = self.getConfig("password")
|
||||
params['cookiecheck'] = '1'
|
||||
params['submit'] = 'Submit'
|
||||
|
||||
if "enchant" in self.story.getMetadata('section'):
|
||||
loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
|
||||
else:
|
||||
loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
|
||||
logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
params['penname']))
|
||||
|
||||
d = self._fetchUrl(loginUrl, params)
|
||||
|
||||
if "Member Account" not in d : #Member Account
|
||||
logging.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
params['penname']))
|
||||
raise exceptions.FailedToLogin(url,params['penname'])
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# Weirdly, different sites use different warning numbers.
|
||||
# If the title search below fails, there's a good chance
|
||||
# you need a different number. print data at that point
|
||||
# and see what the 'click here to continue' url says.
|
||||
addurl = "&ageconsent=ok&warning=1"
|
||||
else:
|
||||
addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url+addurl
|
||||
logging.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
if self.needToLoginCheck(data):
|
||||
# need to log in for this one.
|
||||
self.performLogin(url)
|
||||
data = self._fetchUrl(url)
|
||||
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logging.debug("URL 2nd try: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = bs.BeautifulSoup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',a.string)
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# Find the chapters:
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = soup.findAll('span',{'class':'label'})
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = labelspan.string
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while not defaultGetattr(value,'class') == 'label':
|
||||
svalue += str(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value)
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
|
||||
for char in chars:
|
||||
self.story.addToList('characters',char.string)
|
||||
|
||||
if 'Genre' in label:
|
||||
genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
|
||||
for genre in genres:
|
||||
self.story.addToList('genre',genre.string)
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
|
||||
for warning in warnings:
|
||||
self.story.addToList('warnings',warning.string)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
|
||||
self.setSeries(series_name, i)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logging.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
|
||||
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
|
||||
|
||||
div = soup.find('div', {'id' : 'story1'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
@@ -0,0 +1,233 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import time
|
||||
import logging
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
from .. import BeautifulSoup as bs
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return HLFictionNetAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class HLFictionNetAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','hlf')
|
||||
|
||||
# If all stories from the site fall into the same category,
|
||||
# the site itself isn't likely to label them as such, so we
|
||||
# do.
|
||||
self.story.addToList("category","Highlander")
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%m/%d/%y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'hlfiction.net'
|
||||
|
||||
def getSiteExampleURLs(self):
|
||||
return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
url = self.url
|
||||
logging.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = bs.BeautifulSoup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
## Title and author
|
||||
a = soup.find('div', {'id' : 'pagetitle'})
|
||||
|
||||
aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',aut['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
|
||||
self.story.setMetadata('author',aut.string)
|
||||
aut.extract()
|
||||
|
||||
self.story.setMetadata('title',a.string[:(len(a.string)-3)])
|
||||
|
||||
# Find the chapters:
|
||||
chapters=soup.find('select')
|
||||
if chapters != None:
|
||||
for chapter in chapters.findAll('option'):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
|
||||
else:
|
||||
self.chapterUrls.append((self.story.getMetadata('title'),url))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
|
||||
|
||||
for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
|
||||
a = list.find('a')
|
||||
if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
|
||||
break
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d,k):
|
||||
try:
|
||||
return d[k]
|
||||
except:
|
||||
return ""
|
||||
|
||||
# <span class="label">Rated:</span> NC-17<br /> etc
|
||||
labels = list.findAll('span', {'class' : 'classification'})
|
||||
for labelspan in labels:
|
||||
label = labelspan.string
|
||||
value = labelspan.nextSibling
|
||||
|
||||
if 'Summary' in label:
|
||||
## Everything until the next span class='label'
|
||||
svalue = ""
|
||||
while not defaultGetattr(value,'class') == 'classification':
|
||||
svalue += str(value)
|
||||
value = value.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
#self.story.setMetadata('description',stripHTML(svalue))
|
||||
|
||||
if 'Rated' in label:
|
||||
self.story.setMetadata('rating', value[:len(value)-2])
|
||||
|
||||
if 'Word count' in label:
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
if 'Categories' in label:
|
||||
cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
if 'Characters' in label:
|
||||
for char in value.string.split(', '):
|
||||
if not 'None' in char:
|
||||
self.story.addToList('characters',char)
|
||||
|
||||
if 'Genre' in label:
|
||||
for genre in value.string.split(', '):
|
||||
if not 'None' in genre:
|
||||
self.story.addToList('genre',genre)
|
||||
|
||||
if 'Warnings' in label:
|
||||
for warning in value.string.split(', '):
|
||||
if not 'None' in warning:
|
||||
self.story.addToList('warnings',warning)
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
|
||||
|
||||
try:
|
||||
# Find Series name from series URL.
|
||||
a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
|
||||
series_name = a.string
|
||||
series_url = 'http://'+self.host+'/'+a['href']
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
|
||||
# can't use ^viewstory...$ in case of higher rated stories with javascript href.
|
||||
storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
|
||||
i=1
|
||||
for a in storyas:
|
||||
# skip 'report this' and 'TOC' links
|
||||
if 'contact.php' not in a['href'] and 'index' not in a['href']:
|
||||
if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
|
||||
self.setSeries(series_name, i)
|
||||
break
|
||||
i+=1
|
||||
|
||||
except:
|
||||
# I find it hard to care if the series parsing fails
|
||||
pass
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logging.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
|
||||
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
|
||||
|
||||
div = soup.find('div', {'id' : 'story'})
|
||||
|
||||
if None == div:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,div)
|
||||
@@ -0,0 +1,216 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2012 Fanficdownloader team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import time
|
||||
import logging
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
from .. import BeautifulSoup as bs
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return NHAMagicalWorldsUsAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','nha')
|
||||
|
||||
# If all stories from the site fall into the same category,
|
||||
# the site itself isn't likely to label them as such, so we
|
||||
# do.
|
||||
self.story.addToList("category","Buffy the Vampire Slayer")
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = " %m/%d/%y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'nha.magical-worlds.us'
|
||||
|
||||
def getSiteExampleURLs(self):
|
||||
return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
url = self.url
|
||||
logging.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = bs.BeautifulSoup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
|
||||
|
||||
try:
|
||||
# in case link points somewhere other than the first chapter
|
||||
a = soup.findAll('option')[1]['value']
|
||||
self.story.setMetadata('storyId',a.split('=',)[1])
|
||||
url = 'http://'+self.host+'/'+a
|
||||
soup = bs.BeautifulSoup(self._fetchUrl(url))
|
||||
except:
|
||||
pass
|
||||
|
||||
for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
|
||||
a = info.find('a')
|
||||
if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
|
||||
self.story.setMetadata('title',a.string)
|
||||
break
|
||||
|
||||
|
||||
# Find the chapters:
|
||||
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d$'))
|
||||
if len(chapters) == 0:
|
||||
self.chapterUrls.append((self.story.getMetadata('title'),url))
|
||||
else:
|
||||
for chapter in chapters:
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d):
|
||||
try:
|
||||
return d.name
|
||||
except:
|
||||
return ""
|
||||
|
||||
cats = info.findAll('a',href=re.compile('categories.php'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
|
||||
val = a.nextSibling
|
||||
svalue = ""
|
||||
while not defaultGetattr(val) == 'br':
|
||||
val = val.nextSibling
|
||||
val = val.nextSibling
|
||||
while not defaultGetattr(val) == 'br':
|
||||
svalue += str(val)
|
||||
val = val.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
|
||||
#does not provide convenient way to get word count
|
||||
labels = info.findAll('i')
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = stripHTML(labelspan)
|
||||
|
||||
if 'Rating' in label:
|
||||
self.story.setMetadata('rating', value.split(' -')[0])
|
||||
|
||||
if 'Genres' in label:
|
||||
genres = value.string.split(', ')
|
||||
for genre in genres:
|
||||
if 'None' not in genre:
|
||||
self.story.addToList('genre',genre.split(' -')[0])
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = value.string.split(', ')
|
||||
for char in chars:
|
||||
if 'None' not in char:
|
||||
self.story.addToList('characters',char.split(' -')[0])
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = value.string.split(', ')
|
||||
for warning in warnings:
|
||||
if 'None' not in warning:
|
||||
self.story.addToList('warnings',warning.split(' -')[0])
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
|
||||
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logging.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
data = self._fetchUrl(url)
|
||||
|
||||
soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
|
||||
|
||||
story = soup.find('div', {"id" : "story"})
|
||||
|
||||
if None == story:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return self.utf8FromSoup(url,story)
|
||||
@@ -133,7 +133,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
|
||||
|
||||
## Title
|
||||
a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',a.string)
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
|
||||
+18
-11
@@ -57,18 +57,9 @@
|
||||
<h3>New Sites</h3>
|
||||
<p>
|
||||
|
||||
New sites finestories.com, www.hpfanficarchive.com, svufiction.com, www.twilightarchives.com and www.wizardtales.net.
|
||||
New sites grangerenchanted.com, hlfiction.net and nha.magical-worlds.us.
|
||||
<br />Thanks, Ida!
|
||||
|
||||
</p>
|
||||
<h3>New Feature</h3>
|
||||
<p>
|
||||
|
||||
FFDL now supports multiple authors for AO3, TtH and
|
||||
wraithbait.com. If you know of other sites we support
|
||||
that can have more than one author per story, please let
|
||||
us know.
|
||||
|
||||
</p>
|
||||
<p>
|
||||
Questions? Check out our
|
||||
@@ -78,7 +69,7 @@
|
||||
If you have any problems with this application, please
|
||||
report them in
|
||||
the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>. The
|
||||
<a href="http://4-4-19.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
|
||||
<a href="http://4-4-20.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
|
||||
</p>
|
||||
<div id='error'>
|
||||
{{ error_message }}
|
||||
@@ -499,6 +490,22 @@
|
||||
Use the URL of the story's first chapter, such as
|
||||
<br /><a href="http://www.wizardtales.net/viewstory.php?sid=1234">http://www.wizardtales.net/viewstory.php?sid=1234</a>
|
||||
</dd>
|
||||
<dt>grangerenchanted.com</dt>
|
||||
<dd>
|
||||
Use the URL of the story's chapter list, such as
|
||||
<br /><a href="http://grangerenchanted.com/enchant/viewstory.php?sid=1234">http://grangerenchanted.com/enchant/viewstory.php?sid=1234</a> or
|
||||
<br /><a href="http://malfoymanor.grangerenchanted.com/enchant/viewstory.php?sid=1234">http://grangerenchanted.com/enchant/viewstory.php?sid=1234</a>
|
||||
</dd>
|
||||
<dt>hlfiction.net</dt>
|
||||
<dd>
|
||||
Use the URL of the story's first chapter, such as
|
||||
<br /><a href="http://hlfiction.net/viewstory.php?sid=1234">http://hlfiction.net/viewstory.php?sid=1234</a>
|
||||
</dd>
|
||||
<dt>nha.magical-worlds.us</dt>
|
||||
<dd>
|
||||
Use the URL of the story's chapter list, such as
|
||||
<br /><a href="http://nha.magical-worlds.us/viewstory.php?sid=1234">http://nha.magical-worlds.us/viewstory.php?sid=1234</a>
|
||||
</dd>
|
||||
</dl>
|
||||
<p>
|
||||
A few additional things to know, which will make your life substantially easier:
|
||||
|
||||
@@ -409,6 +409,21 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
[grangerenchanted.com]
|
||||
## Some sites require login (or login for some rated stories) The
|
||||
## program can prompt you, or you can save it in config. In
|
||||
## commandline version, this should go in your personal.ini, not
|
||||
## defaults.ini.
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
## Some sites also require the user to confirm they are adult for
|
||||
## adult content. In commandline version, this should go in your
|
||||
## personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
[hlfiction.net]
|
||||
|
||||
[lumos.sycophanthex.com]
|
||||
## Some sites do not require a login, but do require the user to
|
||||
## confirm they are adult for adult content. In commandline version,
|
||||
@@ -425,6 +440,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
|
||||
## this should go in your personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
[nha.magical-worlds.us]
|
||||
|
||||
[occlumency.sycophanthex.com]
|
||||
## Some sites require login (or login for some rated stories) The
|
||||
## program can prompt you, or you can save it in config. In
|
||||
|
||||
Reference in New Issue
Block a user