From e41c7dabeb5d8b55334a3eff2ebd2b2e54047824 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 11 Sep 2013 10:32:57 -0500 Subject: [PATCH] Fix fimfic dates, add fimfic site specific 'groups' metadata. Bump versions. --- app.yaml | 2 +- calibre-plugin/__init__.py | 2 +- defaults.ini | 3 +- .../adapters/adapter_fimfictionnet.py | 37 +++++++++++++++---- fanficdownloader/adapters/base_adapter.py | 10 ++--- index.html | 12 ++---- plugin-defaults.ini | 3 +- 7 files changed, 44 insertions(+), 25 deletions(-) diff --git a/app.yaml b/app.yaml index 089e1dd..ecd4c0e 100644 --- a/app.yaml +++ b/app.yaml @@ -1,6 +1,6 @@ # ffd-retief-hrd fanfictiondownloader application: fanfictiondownloader -version: 4-4-71 +version: 4-4-72 runtime: python27 api_version: 1 threadsafe: true diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 0ec5593..4b508c2 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase): description = 'UI plugin to download FanFiction stories from various sites.' supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 7, 42) + version = (1, 7, 43) minimum_calibre_version = (0, 8, 57) #: This field defines the GUI plugin class that contains all the code diff --git a/defaults.ini b/defaults.ini index 29b6ad1..1ffd008 100644 --- a/defaults.ini +++ b/defaults.ini @@ -1183,12 +1183,13 @@ extracategories:My Little Pony: Friendship is Magic ## Extra metadata that this adapter knows about. See [dramione.org] ## for examples of how to use them. -extra_valid_entries:likes,dislikes,views,total_views,short_description +extra_valid_entries:likes,dislikes,views,total_views,short_description,groups likes_label:Likes dislikes_label:Dislikes views_label:Highest Single Chapter Views total_views_label:Total Views short_description_label:Short Summary +groups_label:Groups ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, diff --git a/fanficdownloader/adapters/adapter_fimfictionnet.py b/fanficdownloader/adapters/adapter_fimfictionnet.py index b5cd487..be22eff 100644 --- a/fanficdownloader/adapters/adapter_fimfictionnet.py +++ b/fanficdownloader/adapters/adapter_fimfictionnet.py @@ -21,7 +21,8 @@ logger = logging.getLogger(__name__) import re import urllib2 import cookielib as cl -from datetime import datetime +#from datetime import datetime +import dateutil.parser as dparser import json from .. import BeautifulSoup as bs @@ -185,14 +186,30 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): hrstr="
" descdivstr = '
'+descdivstr[descdivstr.index(hrstr)+len(hrstr):] self.setDescription(self.url,descdivstr) - + + # Can't trust dates from API anymore I'm told. # Dates are in Unix time # Take the publish date from the first chapter posted - rawDatePublished = storyMetadata["chapters"][0]["date_modified"] - self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished)) - rawDateUpdated = storyMetadata["date_modified"] - self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated)) - + # rawDatePublished = storyMetadata["chapters"][0]["date_modified"] + # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished)) + # rawDateUpdated = storyMetadata["date_modified"] + # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated)) + + oldestChapter = None + newestChapter = None + # Scan all chapters to find the oldest and newest, on + # FiMFiction it's possible for authors to insert new chapters + # out-of-order or change the dates of earlier ones by editing + # them--That WILL break epub update. + for chapterDate in soup.findAll('span', {'class':'date'}): + chapterDate = dparser.parse(chapterDate.contents[1].strip()) + if oldestChapter == None or chapterDate < oldestChapter: + oldestChapter = chapterDate + if newestChapter == None or chapterDate > newestChapter: + newestChapter = chapterDate + self.story.setMetadata("datePublished", oldestChapter) + self.story.setMetadata("dateUpdated", newestChapter) + chars = soup.find("div", {"class":"inner_data"}) # fimfic stopped putting the char name on or around the char # icon now for some reason. Pull it from the image name with @@ -215,7 +232,11 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): if not isinstance(value,basestring): value = unicode(value) self.story.setMetadata(metakey, value) - + + rawGroupList = soup.find('ul', {'id':'story_group_list'}) + if rawGroupList is not None: + for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}): + self.story.addToList("groups",stripHTML(groupName)) def getChapterText(self, url): logger.debug('Getting chapter text from: %s' % url) diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py index ea2aafd..4ca577d 100644 --- a/fanficdownloader/adapters/base_adapter.py +++ b/fanficdownloader/adapters/base_adapter.py @@ -379,7 +379,7 @@ fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"0 "June":"06","July":"07", "August":"08", "September":"09", "October":"10", "November":"11", "December":"12" } -def makeDate(string,format): +def makeDate(string,dateform): # Surprise! Abstracting this turned out to be more useful than # just saving bytes. @@ -388,10 +388,10 @@ def makeDate(string,format): # there's non-english content. -- ficbook.net now makes that a # lie. It has to do something even more complicated to get # Russian month names correct everywhere. - do_abbrev = "%b" in format + do_abbrev = "%b" in dateform - if "%B" in format or do_abbrev: - format = format.replace("%B","%m").replace("%b","%m") + if "%B" in dateform or do_abbrev: + dateform = dateform.replace("%B","%m").replace("%b","%m") for (name,num) in fullmon.items(): if do_abbrev: name = name[:3] # first three for abbrev @@ -399,5 +399,5 @@ def makeDate(string,format): string = string.replace(name,num) break - return datetime.datetime.strptime(string,format) + return datetime.datetime.strptime(string,dateform) diff --git a/index.html b/index.html index 8a79f93..9e2c1de 100644 --- a/index.html +++ b/index.html @@ -53,19 +53,15 @@

Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites much easier.

-

Changes:

- --> +

fanfiction.net

Fanfiction.net appears to be blocking access from Google diff --git a/plugin-defaults.ini b/plugin-defaults.ini index 7a4b8fc..98ba60e 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -1165,12 +1165,13 @@ extracategories:My Little Pony: Friendship is Magic ## Extra metadata that this adapter knows about. See [dramione.org] ## for examples of how to use them. -extra_valid_entries:likes,dislikes,views,total_views,short_description +extra_valid_entries:likes,dislikes,views,total_views,short_description,groups likes_label:Likes dislikes_label:Dislikes views_label:Highest Single Chapter Views total_views_label:Total Views short_description_label:Short Summary +groups_label:Groups ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version,