mirror of
https://github.com/wassname/FanFicFare.git
synced 2026-09-10 11:40:24 +08:00
Add siye.co.uk. Plugin-add 'view defaults.ini', fix delete other forms, 1.0.4.
This commit is contained in:
Jim Miller
@@ -1,6 +1,6 @@
|
||||
# ffd-retief-hrd fanfictiondownloader
|
||||
application: ffd-retief-hrd
|
||||
version: 4-1-1
|
||||
application: fanfictiondownloader
|
||||
version: 4-2-0
|
||||
runtime: python27
|
||||
api_version: 1
|
||||
threadsafe: true
|
||||
|
||||
@@ -25,7 +25,7 @@ class InterfacePluginDemo(InterfaceActionBase):
|
||||
description = 'UI plugin to download FanFiction stories from various sites.'
|
||||
supported_platforms = ['windows', 'osx', 'linux']
|
||||
author = 'Jim Miller'
|
||||
version = (1, 0, 3)
|
||||
version = (1, 0, 4)
|
||||
minimum_calibre_version = (0, 8, 30)
|
||||
|
||||
# action_menu_clone_qaction = True
|
||||
|
||||
@@ -7,5 +7,5 @@
|
||||
Kovid Goyal's 'The InterfacePlugin Demo' and
|
||||
Grant Drake's 'Count Pages' plugins.</p>
|
||||
|
||||
<p>Requires calibre >= 0.7.53</p>
|
||||
<p>Requires calibre >= 0.8.30</p>
|
||||
|
||||
|
||||
@@ -7,8 +7,8 @@ __license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Jim Miller'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
from PyQt4.Qt import (QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QTextEdit,
|
||||
QComboBox, QCheckBox)
|
||||
from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
|
||||
QTextEdit, QComboBox, QCheckBox, QPushButton)
|
||||
|
||||
from calibre.utils.config import JSONConfig
|
||||
|
||||
@@ -118,6 +118,11 @@ class ConfigWidget(QWidget):
|
||||
self.ini.setLineWrapMode(QTextEdit.NoWrap)
|
||||
self.ini.setText(prefs['personal.ini'])
|
||||
self.l.addWidget(self.ini)
|
||||
|
||||
self.defaults = QPushButton('View Defaults', self)
|
||||
self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
|
||||
self.defaults.clicked.connect(self.show_defaults)
|
||||
self.l.addWidget(self.defaults)
|
||||
|
||||
def save_settings(self):
|
||||
prefs['fileform'] = unicode(self.fileform.currentText())
|
||||
@@ -137,4 +142,31 @@ class ConfigWidget(QWidget):
|
||||
# default next time.
|
||||
del prefs['personal.ini']
|
||||
|
||||
def show_defaults(self):
|
||||
text = get_resources('defaults.ini')
|
||||
ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
|
||||
|
||||
class ShowDefaultsIniDialog(QDialog):
|
||||
|
||||
def __init__(self, icon, text, parent=None):
|
||||
QDialog.__init__(self, parent)
|
||||
self.resize(600, 500)
|
||||
self.l = QVBoxLayout()
|
||||
self.setLayout(self.l)
|
||||
self.label = QLabel("Plugin Defaults (Read-Only)")
|
||||
self.label.setToolTip("These all of the plugin's configurable settings\nand their default settings.")
|
||||
self.setWindowTitle(_('Plugin Defaults'))
|
||||
self.setWindowIcon(icon)
|
||||
self.l.addWidget(self.label)
|
||||
|
||||
self.ini = QTextEdit(self)
|
||||
self.ini.setToolTip("These all of the plugin's configurable settings\nand their default settings.")
|
||||
self.ini.setLineWrapMode(QTextEdit.NoWrap)
|
||||
self.ini.setText(text)
|
||||
self.ini.setReadOnly(True)
|
||||
self.l.addWidget(self.ini)
|
||||
|
||||
self.ok_button = QPushButton('OK', self)
|
||||
self.ok_button.clicked.connect(self.hide)
|
||||
self.l.addWidget(self.ok_button)
|
||||
|
||||
|
||||
@@ -104,19 +104,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
|
||||
if not url_list_text:
|
||||
url_list_text = from_second_func()
|
||||
|
||||
#'''http://test1.com?sid=6
|
||||
#''')
|
||||
# http://test1.com?sid=6701
|
||||
# http://test1.com?sid=6702
|
||||
# http://test1.com?sid=6703
|
||||
# http://test1.com?sid=6704
|
||||
# http://test1.com?sid=6705
|
||||
# http://test1.com?sid=6706
|
||||
# http://test1.com?sid=6707
|
||||
# http://test1.com?sid=6708
|
||||
# http://test1.com?sid=6709
|
||||
|
||||
|
||||
# self.gui is the main calibre GUI. It acts as the gateway to access
|
||||
# all the elements of the calibre user interface, it should also be the
|
||||
# parent of the dialog
|
||||
@@ -138,8 +125,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
|
||||
rows = self.gui.library_view.selectionModel().selectedRows()
|
||||
if rows and prefs['urlsfromselected']:
|
||||
book_ids = self.gui.library_view.get_selected_ids()
|
||||
print("book_ids: %s"%book_ids)
|
||||
#print("book_ids: %s"%book_ids)
|
||||
for book_id in book_ids:
|
||||
#print("book_on_device:%s"%self.db.book_on_device(book_id))
|
||||
identifiers = self.db.get_identifiers(book_id,index_is_id=True)
|
||||
if 'url' in identifiers:
|
||||
# identifiers have :->| in url.
|
||||
@@ -264,6 +252,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
|
||||
if collision == SKIP:
|
||||
raise NotGoingToDownload("Skipping duplicate story.")
|
||||
|
||||
# print("Attempting to add to list.")
|
||||
# rl_plugin = self.gui.iactions['Reading List']
|
||||
# rl_plugin.remove_books_from_list('Send',[book_id],refresh_screen=False)
|
||||
# rl_plugin.add_books_to_list('Send',[book_id],refresh_screen=False)
|
||||
#rl_plugin.view_list('Send')
|
||||
|
||||
if collision == OVERWRITE and len(identicalbooks) > 1:
|
||||
raise NotGoingToDownload("More than one identical books--can't tell which to overwrite.")
|
||||
|
||||
@@ -391,6 +385,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
|
||||
## more than one match? add to first off the list.
|
||||
## Shouldn't happen--we checked above.
|
||||
book_id = identicalbooks.pop()
|
||||
|
||||
if collision == UPDATE:
|
||||
if self.db.has_format(book_id,fileform,index_is_id=True):
|
||||
urlchaptercount = int(story.getMetadata('numChapters'))
|
||||
@@ -437,7 +432,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
|
||||
db.add_format_with_hooks(book_id, fileform, tmp, index_is_id=True)
|
||||
|
||||
# get all formats.
|
||||
if prefs['deleteotherforms']:
|
||||
if prefs['deleteotherforms'] and collision in (OVERWRITE, UPDATE):
|
||||
fmts = set([x.lower() for x in db.formats(book_id, index_is_id=True).split(',')])
|
||||
for fmt in fmts:
|
||||
if fmt != fileform:
|
||||
@@ -446,7 +441,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
|
||||
|
||||
if updatemeta or collision == CALIBREONLY:
|
||||
db.set_metadata(book_id,mi)
|
||||
|
||||
|
||||
else: # no matching, adding new.
|
||||
writer.writeStory(tmp)
|
||||
(notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True)
|
||||
@@ -455,10 +450,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
|
||||
# Otherwise list of books doesn't update right away.
|
||||
if addedcount:
|
||||
self.gui.library_view.model().books_added(addedcount)
|
||||
|
||||
|
||||
self.gui.library_view.model().refresh()
|
||||
#self.gui.library_view.model().research()
|
||||
#self.gui.tags_view.recount()
|
||||
|
||||
del adapter
|
||||
del writer
|
||||
|
||||
@@ -42,6 +42,7 @@ import adapter_tthfanficorg
|
||||
import adapter_twilightednet
|
||||
import adapter_twiwritenet
|
||||
import adapter_whoficcom
|
||||
import adapter_siyecouk
|
||||
|
||||
## This bit of complexity allows adapters to be added by just adding
|
||||
## importing. It eliminates the long if/else clauses we used to need
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2011 Fanficdownloader team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import time
|
||||
import logging
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
from .. import BeautifulSoup as bs
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
|
||||
|
||||
# This function is called by the downloader in all adapter_*.py files
|
||||
# in this dir to register the adapter class. So it needs to be
|
||||
# updated to reflect the class below it. That, plus getSiteDomain()
|
||||
# take care of 'Registering'.
|
||||
def getClass():
|
||||
return SiyeCoUkAdapter # XXX
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8",]# 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
# self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
# self.password = ""
|
||||
# self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','siye') # XXX
|
||||
|
||||
# If all stories from the site fall into the same category,
|
||||
# the site itself isn't likely to label them as such, so we
|
||||
# do.
|
||||
self.story.addToList("category","Harry Potter") # XXX
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = "%Y.%m.%d" # XXX
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'www.siye.co.uk' # XXX
|
||||
|
||||
@classmethod
|
||||
def getAcceptDomains(cls):
|
||||
return ['www.siye.co.uk','siye.co.uk']
|
||||
|
||||
def getSiteExampleURLs(self):
|
||||
return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://")+r"(www\.)?"+re.escape("siye.co.uk/siye/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
# ## Login seems to be reasonably standard across eFiction sites.
|
||||
# def needToLoginCheck(self, data):
|
||||
# if 'Registered Users Only' in data \
|
||||
# or 'There is no such account on our website' in data \
|
||||
# or "That password doesn't match the one in our database" in data:
|
||||
# return True
|
||||
# else:
|
||||
# return False
|
||||
|
||||
# def performLogin(self, url):
|
||||
# params = {}
|
||||
|
||||
# if self.password:
|
||||
# params['penname'] = self.username
|
||||
# params['password'] = self.password
|
||||
# else:
|
||||
# params['penname'] = self.getConfig("username")
|
||||
# params['password'] = self.getConfig("password")
|
||||
# params['cookiecheck'] = '1'
|
||||
# params['submit'] = 'Submit'
|
||||
|
||||
# loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
|
||||
# logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
|
||||
# params['penname']))
|
||||
|
||||
# d = self._fetchUrl(loginUrl, params)
|
||||
|
||||
# if "Member Account" not in d : #Member Account
|
||||
# logging.info("Failed to login to URL %s as %s" % (loginUrl,
|
||||
# params['penname']))
|
||||
# raise exceptions.FailedToLogin(url,params['penname'])
|
||||
# return False
|
||||
# else:
|
||||
# return True
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
# if self.is_adult or self.getConfig("is_adult"):
|
||||
# # Weirdly, different sites use different warning numbers.
|
||||
# # If the title search below fails, there's a good chance
|
||||
# # you need a different number. print data at that point
|
||||
# # and see what the 'click here to continue' url says.
|
||||
# addurl = "&ageconsent=ok&warning=4" # XXX
|
||||
# else:
|
||||
# addurl=""
|
||||
|
||||
# index=1 makes sure we see the story chapter index. Some
|
||||
# sites skip that for one-chapter stories.
|
||||
# Except it doesn't this time. :-/
|
||||
url = self.url #+'&index=1'+addurl
|
||||
logging.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
# if self.needToLoginCheck(data):
|
||||
# # need to log in for this one.
|
||||
# self.performLogin(url)
|
||||
# data = self._fetchUrl(url)
|
||||
|
||||
# # The actual text that is used to announce you need to be an
|
||||
# # adult varies from site to site. Again, print data before
|
||||
# # the title search to troubleshoot.
|
||||
# if "Age Consent Required" in data: # XXX
|
||||
# raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
# if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
# raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = bs.BeautifulSoup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/siye/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
|
||||
# need(or easier) to pull other metadata from the author's list page.
|
||||
authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
|
||||
|
||||
## Title
|
||||
titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
|
||||
self.story.setMetadata('title',titlea.string)
|
||||
|
||||
# Find the chapters (from soup, not authsoup):
|
||||
for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href']))
|
||||
|
||||
if self.chapterUrls:
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
else:
|
||||
self.chapterUrls.append((self.story.getMetadata('title'),url))
|
||||
self.story.setMetadata('numChapters',1)
|
||||
|
||||
# The stuff we can get from the chapter list/one-shot page are
|
||||
# in the first table with 95% width.
|
||||
metatable = soup.find('table',{'width':'95%'})
|
||||
|
||||
# Categories
|
||||
cat_as = metatable.findAll('a', href=re.compile(r'categories.php'))
|
||||
for cat_a in cat_as:
|
||||
self.story.addToList('category',stripHTML(cat_a))
|
||||
|
||||
moremetaparts = stripHTML(metatable).split('\n')
|
||||
for part in moremetaparts:
|
||||
part = part.strip()
|
||||
if part.startswith("Characters:"):
|
||||
part = part[part.find(':')+1:]
|
||||
for item in part.split(','):
|
||||
if item.strip() != "None":
|
||||
self.story.addToList('characters',item)
|
||||
|
||||
if part.startswith("Genres:"):
|
||||
part = part[part.find(':')+1:]
|
||||
for item in part.split(','):
|
||||
if item.strip() != "None":
|
||||
self.story.addToList('genre',item)
|
||||
|
||||
if part.startswith("Warnings:"):
|
||||
part = part[part.find(':')+1:]
|
||||
for item in part.split(','):
|
||||
if item.strip() != "None":
|
||||
self.story.addToList('warnings',item)
|
||||
|
||||
if part.startswith("Rating:"):
|
||||
part = part[part.find(':')+1:]
|
||||
self.story.setMetadata('rating',part)
|
||||
|
||||
if part.startswith("Summary:"):
|
||||
part = part[part.find(':')+1:]
|
||||
self.story.setMetadata('description',part)
|
||||
|
||||
|
||||
|
||||
# want to get the next tr of the table.
|
||||
#print("%s"%titlea.parent.parent.findNextSibling('tr'))
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
moremeta = stripHTML(titlea.parent.parent.findNextSibling('tr'))
|
||||
for part in moremeta.replace(' - ','\n').split('\n'):
|
||||
#print("part:%s"%part)
|
||||
try:
|
||||
(name,value) = part.split(': ')
|
||||
except:
|
||||
# not going to worry about fancier processing for the bits
|
||||
# that don't match.
|
||||
continue
|
||||
name=name.strip()
|
||||
value=value.strip()
|
||||
if name == 'Published':
|
||||
self.story.setMetadata('datePublished', makeDate(value, self.dateformat))
|
||||
if name == 'Updated':
|
||||
self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
|
||||
if name == 'Completed':
|
||||
if value == 'Yes':
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
if name == 'Words':
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logging.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
soup = bs.BeautifulSoup(self._fetchUrl(url))
|
||||
|
||||
# not the most unique thing in the work, bit it appears to be
|
||||
# the best we can do there.
|
||||
story = soup.find('span', {'style' : 'font-size: 100%;'})
|
||||
|
||||
if None == story:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
return utf8FromSoup(story)
|
||||
@@ -53,11 +53,12 @@ class Story:
|
||||
def addToList(self,listname,value):
|
||||
if value==None:
|
||||
return
|
||||
value = conditionalRemoveEntities(value)
|
||||
if not self.listables.has_key(listname):
|
||||
self.listables[listname]=[]
|
||||
# prevent duplicates.
|
||||
if not value in self.listables[listname]:
|
||||
self.listables[listname].append(conditionalRemoveEntities(value))
|
||||
self.listables[listname].append(value)
|
||||
|
||||
def getList(self,listname):
|
||||
if not self.listables.has_key(listname):
|
||||
|
||||
+7
-8
@@ -61,14 +61,8 @@
|
||||
considers Python 2.7 Experimental still, so there may be issues.
|
||||
</p>
|
||||
<p>
|
||||
<b>Good news!</b><br />
|
||||
The issue that was causing problems with downloading large stories
|
||||
has been fixed.
|
||||
</p>
|
||||
<p>
|
||||
<b>New Feature</b><br /> You can now set a custom
|
||||
parameter for background_color that will be used with html
|
||||
and epub output. (Note: many epub readers ignore the bg color.)
|
||||
<b>New Site</b><br />
|
||||
Now supporting www.siye.co.uk.
|
||||
</p>
|
||||
<p>
|
||||
If you have any problems with this application, please
|
||||
@@ -215,6 +209,11 @@
|
||||
<br /><a href="http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm">http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm</a>.
|
||||
<br /><a href="http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm">http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm</a>.
|
||||
</dd>
|
||||
<dt>www.siye.co.uk</dt>
|
||||
<dd>
|
||||
Use the URL of the story's chapter list, such as
|
||||
<br /><a href="http://www.siye.co.uk/siye/viewstory.php?sid=123">http://www.siye.co.uk/siye/viewstory.php?sid=123</a>.
|
||||
</dd>
|
||||
</dl>
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user