From f472c75b8c620384cdebcdbd5905b0d5abe7360e Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 18 Jun 2012 12:24:50 -0500 Subject: [PATCH] Fix series# with javascript and different warning#s. geturls w/javascript. --- calibre-plugin/__init__.py | 2 +- .../adapters/adapter_erosnsapphosycophanthexcom.py | 12 +++++++----- fanficdownloader/adapters/adapter_ksarchivecom.py | 12 +++++++----- .../adapters/adapter_libraryofmoriacom.py | 12 +++++++----- .../adapters/adapter_midnightwhispersca.py | 10 ++++++---- fanficdownloader/adapters/adapter_mugglenetcom.py | 12 +++++++----- .../adapters/adapter_nfacommunitycom.py | 12 +++++++----- .../adapters/adapter_yourfanfictioncom.py | 14 +++++++++----- fanficdownloader/geturls.py | 8 +++++++- 9 files changed, 58 insertions(+), 36 deletions(-) diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 87ff614..951c0c3 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase): description = 'UI plugin to download FanFiction stories from various sites.' supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 5, 34) + version = (1, 5, 36) minimum_calibre_version = (0, 8, 30) #: This field defines the GUI plugin class that contains all the code diff --git a/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py b/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py index 1f5567c..b761344 100644 --- a/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py @@ -227,13 +227,15 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter): # use BeautifulSoup HTML parser to make everything easier to find. seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) - storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) i=1 for a in storyas: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - break - i+=1 + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + break + i+=1 except: # I find it hard to care if the series parsing fails diff --git a/fanficdownloader/adapters/adapter_ksarchivecom.py b/fanficdownloader/adapters/adapter_ksarchivecom.py index 90bc6a2..5a38899 100644 --- a/fanficdownloader/adapters/adapter_ksarchivecom.py +++ b/fanficdownloader/adapters/adapter_ksarchivecom.py @@ -274,13 +274,15 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX # use BeautifulSoup HTML parser to make everything easier to find. seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) - storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) i=1 for a in storyas: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - break - i+=1 + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + break + i+=1 except: # I find it hard to care if the series parsing fails diff --git a/fanficdownloader/adapters/adapter_libraryofmoriacom.py b/fanficdownloader/adapters/adapter_libraryofmoriacom.py index 04c9bf0..942d7ea 100644 --- a/fanficdownloader/adapters/adapter_libraryofmoriacom.py +++ b/fanficdownloader/adapters/adapter_libraryofmoriacom.py @@ -222,13 +222,15 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter): # use BeautifulSoup HTML parser to make everything easier to find. seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) - storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) i=1 for a in storyas: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - break - i+=1 + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + break + i+=1 except: # I find it hard to care if the series parsing fails diff --git a/fanficdownloader/adapters/adapter_midnightwhispersca.py b/fanficdownloader/adapters/adapter_midnightwhispersca.py index 8e48fb0..da7ad4c 100644 --- a/fanficdownloader/adapters/adapter_midnightwhispersca.py +++ b/fanficdownloader/adapters/adapter_midnightwhispersca.py @@ -260,10 +260,12 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) i=1 for a in storyas: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - break - i+=1 + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + break + i+=1 except: # I find it hard to care if the series parsing fails diff --git a/fanficdownloader/adapters/adapter_mugglenetcom.py b/fanficdownloader/adapters/adapter_mugglenetcom.py index 4325c21..f9edda4 100644 --- a/fanficdownloader/adapters/adapter_mugglenetcom.py +++ b/fanficdownloader/adapters/adapter_mugglenetcom.py @@ -303,13 +303,15 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX # use BeautifulSoup HTML parser to make everything easier to find. seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) - storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) i=1 for a in storyas: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - break - i+=1 + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + break + i+=1 except: # I find it hard to care if the series parsing fails diff --git a/fanficdownloader/adapters/adapter_nfacommunitycom.py b/fanficdownloader/adapters/adapter_nfacommunitycom.py index 0795145..e33d83d 100644 --- a/fanficdownloader/adapters/adapter_nfacommunitycom.py +++ b/fanficdownloader/adapters/adapter_nfacommunitycom.py @@ -261,13 +261,15 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX # use BeautifulSoup HTML parser to make everything easier to find. seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) - storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) i=1 for a in storyas: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - break - i+=1 + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + break + i+=1 except: # I find it hard to care if the series parsing fails diff --git a/fanficdownloader/adapters/adapter_yourfanfictioncom.py b/fanficdownloader/adapters/adapter_yourfanfictioncom.py index 6b618d5..b52b7e0 100644 --- a/fanficdownloader/adapters/adapter_yourfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_yourfanfictioncom.py @@ -234,13 +234,17 @@ class YourFanfictionComAdapter(BaseSiteAdapter): # use BeautifulSoup HTML parser to make everything easier to find. seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) - storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) i=1 for a in storyas: - if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): - self.setSeries(series_name, i) - break - i+=1 + print("series a['href']:%s"%a['href']) + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + break + i+=1 except: # I find it hard to care if the series parsing fails diff --git a/fanficdownloader/geturls.py b/fanficdownloader/geturls.py index 65bf939..fc7171e 100644 --- a/fanficdownloader/geturls.py +++ b/fanficdownloader/geturls.py @@ -15,7 +15,7 @@ # limitations under the License. # - +import re import urlparse import urllib2 as u2 import ConfigParser @@ -37,7 +37,13 @@ def get_urls_from_page(url): for a in soup.findAll('a'): if a.has_key('href'): href = form_url(url,a['href']) + # lots of eFiction sites use similar 'are you old enough' javascript links. + if 'javascript' in a['href'] and 'viewstory.php' in a['href']: + m = re.search(r"'(?P(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href']) + if m != None: + href = form_url(url,m.group('sid')) try: + href = href.replace('&index=1','') adapter = adapters.getAdapter(config,href,"EPUB") if adapter.story.getMetadata('storyUrl') not in normalized: normalized.add(adapter.story.getMetadata('storyUrl'))