From 3488c54f3dc7bbd9a9f55244a87783e28c2cc335 Mon Sep 17 00:00:00 2001 From: cryzed Date: Mon, 31 Aug 2015 06:05:25 +0200 Subject: [PATCH 1/2] Added adapter for quotev.com --- calibre-plugin/plugin-defaults.ini | 2 + fanficfare/adapters/__init__.py | 1 + fanficfare/adapters/adapter_quotevcom.py | 99 ++++++++++++++++++++++++ fanficfare/defaults.ini | 2 + 4 files changed, 104 insertions(+) create mode 100644 fanficfare/adapters/adapter_quotevcom.py diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index a2f9b60..51f1187 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -1938,6 +1938,8 @@ extracategories:Psych ## Site dedicated to these categories/characters/ships extracategories:Queer as Folk +[quotev.com] + [www.restrictedsection.org] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In diff --git a/fanficfare/adapters/__init__.py b/fanficfare/adapters/__init__.py index 43505b7..fb0d78d 100644 --- a/fanficfare/adapters/__init__.py +++ b/fanficfare/adapters/__init__.py @@ -140,6 +140,7 @@ import adapter_forumssufficientvelocitycom import adapter_questionablequestingcom import adapter_ninelivesdarksolaceorg import adapter_masseffect2in +import adapter_quotevcom ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff --git a/fanficfare/adapters/adapter_quotevcom.py b/fanficfare/adapters/adapter_quotevcom.py new file mode 100644 index 0000000..e95fde1 --- /dev/null +++ b/fanficfare/adapters/adapter_quotevcom.py @@ -0,0 +1,99 @@ +import re +import urlparse +import urllib2 +import datetime + +from .. import exceptions +from base_adapter import BaseSiteAdapter + +SITE_DOMAIN = 'quotev.com' +STORY_URL_TEMPLATE = 'http://www.quotev.com/story/%s' + + +def getClass(): + return QuotevComAdapter + + +def get_url_path_segments(url): + return tuple(filter(None, url.split('/')[3:])) + + +# TODO: Possibly add pages/readers/reads/favorites +class QuotevComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + story_id = get_url_path_segments(url)[1] + self._setURL(STORY_URL_TEMPLATE % story_id) + self.story.setMetadata('storyId', story_id) + self.story.setMetadata('siteabbrev', SITE_DOMAIN) + + @staticmethod + def getSiteDomain(): + return SITE_DOMAIN + + @classmethod + def getSiteExampleURLs(cls): + return STORY_URL_TEMPLATE % '1234' + + def getSiteURLPattern(self): + pattern = re.escape(STORY_URL_TEMPLATE.rsplit('%', 1)[0]) + r'(.+?)($|&|/)' + pattern = pattern.replace(r'http\:', r'https?\:') + pattern = pattern.replace(r'https?\:\/\/www\.', r'https?\:\/\/(www\.)?') + return pattern + + def use_pagecache(self): + return True + + def extractChapterUrlsAndMetadata(self): + try: + data = self._fetchUrl(self.url) + except urllib2.HTTPError: + raise exceptions.FailedToDownload(self.url) + + soup = self.make_soup(data) + + element = soup.find('div', {'class': 'result_head'}) + if not element: + raise exceptions.StoryDoesNotExist(self.url) + + self.story.setMetadata('title', element.find('span', recursive=False).get_text()) + + element = soup.find('div', {'class': 'desc_creator'})('a')[1] + self.story.setMetadata('author', element.get_text()) + self.story.setMetadata('authorId', get_url_path_segments(element['href'])[0]) + self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, element['href'])) + self.setDescription(self.url, self.utf8FromSoup(self.url, soup.find('div', id='qdesct'))) + self.setCoverImage(self.url, urlparse.urljoin(self.url, soup.find('img', {'class': 'logo'})['src'])) + + for a in soup.find('div', {'class': 'tag'})('a'): + if a['href'] == '#': + continue + + self.story.addToList('category', a.get_text()) + + self.story.setMetadata( + 'status', 'Completed' if 'completed' in + soup.find('div', {'class': 't'})('div', recursive=False)[1].div.get_text() + else 'In-Progress' + ) + + elements = soup('span', {'class': 'q_time'}) + self.story.setMetadata('datePublished', datetime.datetime.fromtimestamp(float(elements[0]['ts']))) + self.story.setMetadata('dateUpdated', datetime.datetime.fromtimestamp(float(elements[1]['ts']))) + + for a in soup.find('div', id='rselect')('a'): + self.chapterUrls.append((a.get_text(), urlparse.urljoin(self.url, a['href']))) + + self.story.setMetadata('numChapters', len(self.chapterUrls)) + + def getChapterText(self, url): + data = self._fetchUrl(url) + soup = self.make_soup(data) + + element = soup.find('div', id='restxt') + for a in element('a'): + a.unwrap() + + return self.utf8FromSoup(url, element) diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 89e6320..12dfa9f 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -1918,6 +1918,8 @@ extracategories:Psych ## Site dedicated to these categories/characters/ships extracategories:Queer as Folk +[quotev.com] + [www.restrictedsection.org] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In From 4cb69a0a3c9dfd5b56cc6972766ff3ff752e84fc Mon Sep 17 00:00:00 2001 From: cryzed Date: Mon, 31 Aug 2015 14:18:06 +0200 Subject: [PATCH 2/2] Parse additional metadata with the quotev adapter --- calibre-plugin/plugin-defaults.ini | 5 +++++ fanficfare/adapters/adapter_quotevcom.py | 25 ++++++++++++++++-------- fanficfare/defaults.ini | 5 +++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index 51f1187..a02f1be 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -1939,6 +1939,11 @@ extracategories:Psych extracategories:Queer as Folk [quotev.com] +extra_valid_entries:pages,readers,reads,favorites +pages_label:Pages +readers_label:Readers +reads_label:Reads +favorites_label:Favorites [www.restrictedsection.org] ## Some sites require login (or login for some rated stories) The diff --git a/fanficfare/adapters/adapter_quotevcom.py b/fanficfare/adapters/adapter_quotevcom.py index e95fde1..b4fe52b 100644 --- a/fanficfare/adapters/adapter_quotevcom.py +++ b/fanficfare/adapters/adapter_quotevcom.py @@ -18,7 +18,6 @@ def get_url_path_segments(url): return tuple(filter(None, url.split('/')[3:])) -# TODO: Possibly add pages/readers/reads/favorites class QuotevComAdapter(BaseSiteAdapter): def __init__(self, config, url): @@ -64,7 +63,7 @@ class QuotevComAdapter(BaseSiteAdapter): self.story.setMetadata('author', element.get_text()) self.story.setMetadata('authorId', get_url_path_segments(element['href'])[0]) self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, element['href'])) - self.setDescription(self.url, self.utf8FromSoup(self.url, soup.find('div', id='qdesct'))) + self.setDescription(self.url, soup.find('div', id='qdesct')) self.setCoverImage(self.url, urlparse.urljoin(self.url, soup.find('img', {'class': 'logo'})['src'])) for a in soup.find('div', {'class': 'tag'})('a'): @@ -73,12 +72,6 @@ class QuotevComAdapter(BaseSiteAdapter): self.story.addToList('category', a.get_text()) - self.story.setMetadata( - 'status', 'Completed' if 'completed' in - soup.find('div', {'class': 't'})('div', recursive=False)[1].div.get_text() - else 'In-Progress' - ) - elements = soup('span', {'class': 'q_time'}) self.story.setMetadata('datePublished', datetime.datetime.fromtimestamp(float(elements[0]['ts']))) self.story.setMetadata('dateUpdated', datetime.datetime.fromtimestamp(float(elements[1]['ts']))) @@ -88,6 +81,22 @@ class QuotevComAdapter(BaseSiteAdapter): self.story.setMetadata('numChapters', len(self.chapterUrls)) + element = soup.find('div', {'class': 't'})('div', recursive=False)[1].div + data = filter(None, (x.strip() for x in element('span')[1].next_sibling.split(u'\xb7'))) + + if 'completed' in data: + self.story.setMetadata('status', 'Completed') + data.remove('completed') + else: + self.story.setMetadata('status', 'In-Progress') + + for datum in data: + value, key = datum.split() + self.story.setMetadata(key, value.replace(',', '').replace('.', '')) + + self.story.setMetadata('favorites', soup.find('div', id='favqn').get_text()) + self.story.setMetadata('comments', soup.find('a', id='comment_btn').span.get_text()) + def getChapterText(self, url): data = self._fetchUrl(url) soup = self.make_soup(data) diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 12dfa9f..1b05cbc 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -1919,6 +1919,11 @@ extracategories:Psych extracategories:Queer as Folk [quotev.com] +extra_valid_entries:pages,readers,reads,favorites +pages_label:Pages +readers_label:Readers +reads_label:Reads +favorites_label:Favorites [www.restrictedsection.org] ## Some sites require login (or login for some rated stories) The