Requires calibre >= 0.7.53
+Requires calibre >= 0.8.30
diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index 6b78640..8fdb9bd 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -7,8 +7,8 @@ __license__ = 'GPL v3' __copyright__ = '2011, Jim Miller' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import (QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QTextEdit, - QComboBox, QCheckBox) +from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, + QTextEdit, QComboBox, QCheckBox, QPushButton) from calibre.utils.config import JSONConfig @@ -118,6 +118,11 @@ class ConfigWidget(QWidget): self.ini.setLineWrapMode(QTextEdit.NoWrap) self.ini.setText(prefs['personal.ini']) self.l.addWidget(self.ini) + + self.defaults = QPushButton('View Defaults', self) + self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.") + self.defaults.clicked.connect(self.show_defaults) + self.l.addWidget(self.defaults) def save_settings(self): prefs['fileform'] = unicode(self.fileform.currentText()) @@ -137,4 +142,31 @@ class ConfigWidget(QWidget): # default next time. del prefs['personal.ini'] + def show_defaults(self): + text = get_resources('defaults.ini') + ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_() + +class ShowDefaultsIniDialog(QDialog): + def __init__(self, icon, text, parent=None): + QDialog.__init__(self, parent) + self.resize(600, 500) + self.l = QVBoxLayout() + self.setLayout(self.l) + self.label = QLabel("Plugin Defaults (Read-Only)") + self.label.setToolTip("These all of the plugin's configurable settings\nand their default settings.") + self.setWindowTitle(_('Plugin Defaults')) + self.setWindowIcon(icon) + self.l.addWidget(self.label) + + self.ini = QTextEdit(self) + self.ini.setToolTip("These all of the plugin's configurable settings\nand their default settings.") + self.ini.setLineWrapMode(QTextEdit.NoWrap) + self.ini.setText(text) + self.ini.setReadOnly(True) + self.l.addWidget(self.ini) + + self.ok_button = QPushButton('OK', self) + self.ok_button.clicked.connect(self.hide) + self.l.addWidget(self.ok_button) + diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index 2570764..4adbea7 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -104,19 +104,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if not url_list_text: url_list_text = from_second_func() - #'''http://test1.com?sid=6 -#''') -# http://test1.com?sid=6701 -# http://test1.com?sid=6702 -# http://test1.com?sid=6703 -# http://test1.com?sid=6704 -# http://test1.com?sid=6705 -# http://test1.com?sid=6706 -# http://test1.com?sid=6707 -# http://test1.com?sid=6708 -# http://test1.com?sid=6709 - - # self.gui is the main calibre GUI. It acts as the gateway to access # all the elements of the calibre user interface, it should also be the # parent of the dialog @@ -138,8 +125,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction): rows = self.gui.library_view.selectionModel().selectedRows() if rows and prefs['urlsfromselected']: book_ids = self.gui.library_view.get_selected_ids() - print("book_ids: %s"%book_ids) + #print("book_ids: %s"%book_ids) for book_id in book_ids: + #print("book_on_device:%s"%self.db.book_on_device(book_id)) identifiers = self.db.get_identifiers(book_id,index_is_id=True) if 'url' in identifiers: # identifiers have :->| in url. @@ -264,6 +252,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if collision == SKIP: raise NotGoingToDownload("Skipping duplicate story.") + # print("Attempting to add to list.") + # rl_plugin = self.gui.iactions['Reading List'] + # rl_plugin.remove_books_from_list('Send',[book_id],refresh_screen=False) + # rl_plugin.add_books_to_list('Send',[book_id],refresh_screen=False) + #rl_plugin.view_list('Send') + if collision == OVERWRITE and len(identicalbooks) > 1: raise NotGoingToDownload("More than one identical books--can't tell which to overwrite.") @@ -391,6 +385,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): ## more than one match? add to first off the list. ## Shouldn't happen--we checked above. book_id = identicalbooks.pop() + if collision == UPDATE: if self.db.has_format(book_id,fileform,index_is_id=True): urlchaptercount = int(story.getMetadata('numChapters')) @@ -437,7 +432,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): db.add_format_with_hooks(book_id, fileform, tmp, index_is_id=True) # get all formats. - if prefs['deleteotherforms']: + if prefs['deleteotherforms'] and collision in (OVERWRITE, UPDATE): fmts = set([x.lower() for x in db.formats(book_id, index_is_id=True).split(',')]) for fmt in fmts: if fmt != fileform: @@ -446,7 +441,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if updatemeta or collision == CALIBREONLY: db.set_metadata(book_id,mi) - + else: # no matching, adding new. writer.writeStory(tmp) (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True) @@ -455,10 +450,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction): # Otherwise list of books doesn't update right away. if addedcount: self.gui.library_view.model().books_added(addedcount) - + self.gui.library_view.model().refresh() - #self.gui.library_view.model().research() - #self.gui.tags_view.recount() del adapter del writer diff --git a/fanficdownloader/adapters/__init__.py b/fanficdownloader/adapters/__init__.py index 70f5006..89ef2c9 100644 --- a/fanficdownloader/adapters/__init__.py +++ b/fanficdownloader/adapters/__init__.py @@ -42,6 +42,7 @@ import adapter_tthfanficorg import adapter_twilightednet import adapter_twiwritenet import adapter_whoficcom +import adapter_siyecouk ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff --git a/fanficdownloader/adapters/adapter_siyecouk.py b/fanficdownloader/adapters/adapter_siyecouk.py new file mode 100644 index 0000000..8222b15 --- /dev/null +++ b/fanficdownloader/adapters/adapter_siyecouk.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return SiyeCoUkAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SiyeCoUkAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8",]# 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + # self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + # self.password = "" + # self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + logging.debug("storyId: (%s)"%self.story.getMetadata('storyId')) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','siye') # XXX + + # If all stories from the site fall into the same category, + # the site itself isn't likely to label them as such, so we + # do. + self.story.addToList("category","Harry Potter") # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y.%m.%d" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.siye.co.uk' # XXX + + @classmethod + def getAcceptDomains(cls): + return ['www.siye.co.uk','siye.co.uk'] + + def getSiteExampleURLs(self): + return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+r"(www\.)?"+re.escape("siye.co.uk/siye/viewstory.php?sid=")+r"\d+$" + + # ## Login seems to be reasonably standard across eFiction sites. + # def needToLoginCheck(self, data): + # if 'Registered Users Only' in data \ + # or 'There is no such account on our website' in data \ + # or "That password doesn't match the one in our database" in data: + # return True + # else: + # return False + + # def performLogin(self, url): + # params = {} + + # if self.password: + # params['penname'] = self.username + # params['password'] = self.password + # else: + # params['penname'] = self.getConfig("username") + # params['password'] = self.getConfig("password") + # params['cookiecheck'] = '1' + # params['submit'] = 'Submit' + + # loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + # logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + # params['penname'])) + + # d = self._fetchUrl(loginUrl, params) + + # if "Member Account" not in d : #Member Account + # logging.info("Failed to login to URL %s as %s" % (loginUrl, + # params['penname'])) + # raise exceptions.FailedToLogin(url,params['penname']) + # return False + # else: + # return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # if self.is_adult or self.getConfig("is_adult"): + # # Weirdly, different sites use different warning numbers. + # # If the title search below fails, there's a good chance + # # you need a different number. print data at that point + # # and see what the 'click here to continue' url says. + # addurl = "&ageconsent=ok&warning=4" # XXX + # else: + # addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + # Except it doesn't this time. :-/ + url = self.url #+'&index=1'+addurl + logging.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # if self.needToLoginCheck(data): + # # need to log in for this one. + # self.performLogin(url) + # data = self._fetchUrl(url) + + # # The actual text that is used to announce you need to be an + # # adult varies from site to site. Again, print data before + # # the title search to troubleshoot. + # if "Age Consent Required" in data: # XXX + # raise exceptions.AdultCheckRequired(self.url) + + # if "Access denied. This story has not been validated by the adminstrators of this site." in data: + # raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/siye/'+a['href']) + self.story.setMetadata('author',a.string) + + # need(or easier) to pull other metadata from the author's list page. + authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + ## Title + titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',titlea.string) + + # Find the chapters (from soup, not authsoup): + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href'])) + + if self.chapterUrls: + self.story.setMetadata('numChapters',len(self.chapterUrls)) + else: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.story.setMetadata('numChapters',1) + + # The stuff we can get from the chapter list/one-shot page are + # in the first table with 95% width. + metatable = soup.find('table',{'width':'95%'}) + + # Categories + cat_as = metatable.findAll('a', href=re.compile(r'categories.php')) + for cat_a in cat_as: + self.story.addToList('category',stripHTML(cat_a)) + + moremetaparts = stripHTML(metatable).split('\n') + for part in moremetaparts: + part = part.strip() + if part.startswith("Characters:"): + part = part[part.find(':')+1:] + for item in part.split(','): + if item.strip() != "None": + self.story.addToList('characters',item) + + if part.startswith("Genres:"): + part = part[part.find(':')+1:] + for item in part.split(','): + if item.strip() != "None": + self.story.addToList('genre',item) + + if part.startswith("Warnings:"): + part = part[part.find(':')+1:] + for item in part.split(','): + if item.strip() != "None": + self.story.addToList('warnings',item) + + if part.startswith("Rating:"): + part = part[part.find(':')+1:] + self.story.setMetadata('rating',part) + + if part.startswith("Summary:"): + part = part[part.find(':')+1:] + self.story.setMetadata('description',part) + + + + # want to get the next tr of the table. + #print("%s"%titlea.parent.parent.findNextSibling('tr')) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + moremeta = stripHTML(titlea.parent.parent.findNextSibling('tr')) + for part in moremeta.replace(' - ','\n').split('\n'): + #print("part:%s"%part) + try: + (name,value) = part.split(': ') + except: + # not going to worry about fancier processing for the bits + # that don't match. + continue + name=name.strip() + value=value.strip() + if name == 'Published': + self.story.setMetadata('datePublished', makeDate(value, self.dateformat)) + if name == 'Updated': + self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat)) + if name == 'Completed': + if value == 'Yes': + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + if name == 'Words': + self.story.setMetadata('numWords', value) + + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logging.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url)) + + # not the most unique thing in the work, bit it appears to be + # the best we can do there. + story = soup.find('span', {'style' : 'font-size: 100%;'}) + + if None == story: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return utf8FromSoup(story) diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index 68b24be..2768854 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -53,11 +53,12 @@ class Story: def addToList(self,listname,value): if value==None: return + value = conditionalRemoveEntities(value) if not self.listables.has_key(listname): self.listables[listname]=[] # prevent duplicates. if not value in self.listables[listname]: - self.listables[listname].append(conditionalRemoveEntities(value)) + self.listables[listname].append(value) def getList(self,listname): if not self.listables.has_key(listname): diff --git a/index.html b/index.html index 9b10d0b..bcde2fb 100644 --- a/index.html +++ b/index.html @@ -61,14 +61,8 @@ considers Python 2.7 Experimental still, so there may be issues.
- Good news!
- The issue that was causing problems with downloading large stories
- has been fixed.
-
- New Feature
You can now set a custom
- parameter for background_color that will be used with html
- and epub output. (Note: many epub readers ignore the bg color.)
+ New Site
+ Now supporting www.siye.co.uk.
If you have any problems with this application, please
@@ -215,6 +209,11 @@
http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm.
http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm.
+