Update CLI download zip.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-5-00
+version: 4-5-04
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 19)
+    version             = (1, 8, 23)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -2004,7 +2004,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         # cust cols can convert back to numbers and
                         # add.
                         book['anthology_meta_list'][k]=True
-    
+
+        print("book['url']:%s"%book['url'])
+        configuration = get_ffdl_config(book['url'],fileform)
         if existingbook:
             book['title'] = deftitle = existingbook['title']
             book['comments'] = existingbook['comments']
@@ -2023,7 +2025,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         book['title'] = deftitle
                         break
             
-            configuration = get_ffdl_config(book['url'],fileform)
             logger.debug("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
             if configuration.getConfig('anthology_title_pattern'):
                 tmplt = Template(configuration.getConfig('anthology_title_pattern'))
@@ -2039,7 +2040,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         for v in ['Completed','In-Progress']:
             if v in book['tags']:
                 book['tags'].remove(v)
-        book['tags'].append('Anthology')
+        book['tags'].extend(configuration.getConfigList('anthology_tags'))
         book['all_metadata']['anthology'] = "true"
         
         return book

defaults.ini

@@ -626,6 +626,29 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[bloodshedverse.com]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:warnings,reviews
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
+## Strips links found in the story text
+## Specific to bloodshedverse.com
+strip_text_links:true
+
 [bloodties-fans.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -817,6 +840,14 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fictionpad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1063,6 +1094,16 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[spikeluver.com]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:warnings,reviews
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [stargate-atlantis.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
@@ -1161,6 +1202,13 @@ awards_label:Awards
 
 cover_exclusion_regexp:art/.*Awards.jpg
 
+[voracity2.e-fic.com]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,readings
+reviews_label:Reviews
+readings_label:Readings
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1313,14 +1361,6 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[ficwad.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
 [www.fimfiction.net]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,

downloader.py

@@ -23,6 +23,7 @@ import getpass
 import string
 import ConfigParser
 from subprocess import call
+import pprint
 
 import logging
 if sys.version_info >= (2, 7):
@@ -292,7 +293,7 @@ def main(argv,
        else:
            # regular download
            if options.metaonly:
-               print adapter.getStoryMetadataOnly().getAllMetadata()
+               pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
            
            output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        

fanficdownloader/adapters/__init__.py

@@ -123,6 +123,9 @@ import adapter_fictionpadcom
 import adapter_storiesonlinenet
 import adapter_trekiverseorg
 import adapter_literotica
+import adapter_voracity2eficcom
+import adapter_spikeluvercom
+import adapter_bloodshedversecom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -206,7 +209,7 @@ def getClassFor(url):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.
     fixedurl = re.sub(r"#.*$","",fixedurl)
-    
+
     parsedUrl = up.urlparse(fixedurl)
     domain = parsedUrl.netloc.lower()
     if( domain != parsedUrl.netloc ):

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -145,10 +145,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
+                raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
+        if "Sorry, we couldn't find the work you were looking for." in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+            
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url,data)

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -0,0 +1,187 @@
+from datetime import timedelta
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return BloodshedverseComAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class BloodshedverseComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'bvc'
+    SITE_DOMAIN = 'bloodshedverse.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
+
+    STARTED_DATETIME_FORMAT = '%m/%d/%Y'
+    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_no = query_data['no'][0]
+
+        self.story.setMetadata('storyId', story_no)
+        self._setURL(self.READ_URL_TEMPLATE % story_no)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception, parameters=None):
+        try:
+            data = self._fetchUrl(url, parameters)
+        except urllib2.HTTPError:
+            raise exception(self.url)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return BloodshedverseComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL + 'stories.php?go=') + r'(read|chapters)\&no=\d+$'
+
+    # Override stripURLParameters so the "no" parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
+
+    def extractChapterUrlsAndMetadata(self):
+        # Use FailedToDownload exception because that's the only error that can
+        # occur here, missing stories don't return a 404 header
+        soup = self._customized_fetch_url(self.url, exceptions.FailedToDownload)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        for option in soup.find('select', {'name': 'chapter'}):
+            title = option.string.strip()
+            url = self.READ_URL_TEMPLATE % option['value']
+            self.chapterUrls.append((title, url))
+
+        # Get the URL to the author's page and find the correct story entry to
+        # scrape the metadata
+        author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
+        soup = self._customized_fetch_url(author_url, exceptions.FailedToDownload)
+
+        story_no = self.story.getMetadata('storyId')
+        # Ignore first list_box div, it only contains the author information
+        for list_box in soup('div', {'class': 'list_box'})[1:]:
+            url = list_box.find('a', {'class': 'fictitle'})['href']
+            query_data = _get_query_data(url)
+
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if query_data['no'] == story_no:
+                break
+
+        title_anchor = list_box.find('a', {'class': 'fictitle'})
+        self.story.setMetadata('title', title_anchor.string.strip())
+
+        author_anchor = title_anchor.findNextSibling('a')
+        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
+
+        list_review = list_box.find('div', {'class': 'list_review'})
+        reviews = list_review.a.string.strip().split(' ', 1)[0]
+        self.story.setMetadata('reviews', reviews)
+
+        summary_div = list_box.find('div', {'class': 'list_summary'})
+        if not self.getConfig('keep_summary_html'):
+            summary = ''.join(summary_div(text=True))
+        else:
+            summary = self.utf8FromSoup(author_url, summary_div)
+
+        self.story.setMetadata('description', summary)
+
+        # I'm assuming this to be the category, not sure what else it could be
+        first_listinfo = list_box.find('div', {'class': 'list_info'})
+        self.story.addToList('category', first_listinfo.a.string.strip())
+
+        for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
+            for b_tag in list_info('b'):
+                key = b_tag.string.strip(': ')
+                # Strip colons from the beginning, superfluous spaces and minus
+                # characters from the end, and possibly trailing commas from
+                # the warnings if only one is present
+                value = b_tag.nextSibling.string.strip(': -,')
+
+                if key == 'Genre':
+                    for genre in value.split(', '):
+                        # Ignore the "none" genre
+                        if not genre == 'none':
+                            self.story.addToList('genre', genre)
+
+                elif key == 'Rating':
+                    self.story.setMetadata('rating', value)
+
+                elif key == 'Complete':
+                    self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+                elif key == 'Warning':
+                    for warning in value.split(', '):
+                        # The string here starts with ", " before the actual list
+                        # of values sometimes, so check for an empty warning
+                        # and ignore the "none" warning.
+                        if not warning or warning == 'none':
+                            continue
+
+                        self.story.addToList('warnings', warning)
+
+                elif key == 'Chapters':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Words':
+                    # Apparently only numChapters need to be an integer for
+                    # some strange reason. Remove possible ',' characters as to
+                    # not confuse the codebase down the line
+                    self.story.setMetadata('numWords', value.replace(',', ''))
+
+                elif key == 'Started':
+                    self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
+
+                elif key == 'Updated':
+                    date_string, period = value.rsplit(' ', 1)
+                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
+
+                    # Rather ugly hack to work around Calibre's changing of
+                    # Python's locale setting, causing am/pm to not be properly
+                    # parsed by strptime() when using a non-english locale
+                    if period == 'pm':
+                        date += timedelta(hours=12)
+                    self.story.setMetadata('dateUpdated', date)
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
+        storytext_div = soup.find('div', {'class': 'storytext'})
+
+        if self.getConfig('strip_text_links'):
+            for anchor in storytext_div('a', {'class': 'FAtxtL'}):
+                navigable_string = BeautifulSoup.NavigableString(anchor.string)
+                anchor.replaceWith(navigable_string)
+
+        return self.utf8FromSoup(url, storytext_div)

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -182,6 +182,11 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         # first a tag in pagetitle is title
         self.story.setMetadata('title',stripHTML(div.find('a')))
+        div.find('a').extract()
+        # only thing left in div(pagetitle) now should be 'by' and rating.
+        rating = stripHTML(div)
+        if '[' in rating:
+            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
 
         for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
                                                        self.story.getMetadata('storyId')+'&chapter=\d+')):
@@ -234,31 +239,28 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
                 if 'Yes' in value:

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -133,6 +133,7 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
         author = tables['users'][0]
         story = tables['stories'][0]
         story_ver = tables['story_versions'][0]
+        print("story:%s"%story)
         
         self.story.setMetadata('authorId',author['id'])
         self.story.setMetadata('author',author['display_name'])
@@ -151,7 +152,8 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('comments',story['comments_count'])
         self.story.setMetadata('views',story['views_count'])
         self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
-        self.story.setMetadata('dislikes',int(story['dislikes']))
+        if 'dislikes' in story:
+            self.story.setMetadata('dislikes',int(story['dislikes']))
 
         if story_ver['is_complete']:
             self.story.setMetadata('status', 'Completed')

fanficdownloader/adapters/adapter_literotica.py

@@ -42,16 +42,19 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         
         self.story.setMetadata('siteabbrev','litero')
 
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        # normalize to first chapter.  Not sure if they ever have more than 2 digits.
+        storyid = self.parsedUrl.path.split('/',)[2]
+        if re.match(r'-ch\d\d$',storyid):
+            storyid = storyid[:-2]+'01'
+        self.story.setMetadata('storyId',storyid)
         
         self.origurl = url
-        if "http://www.i." in self.origurl:
+        if "//www.i." in self.origurl:
             ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("http://www.i.","http://www.")
+            self.origurl = self.origurl.replace("//www.i.","//www.")
 
         # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
+        self._setURL(url[:url.index('//')+2]+self.getSiteDomain()\
                          +"/s/"+self.story.getMetadata('storyId'))
 
         # The date format will vary from site to site.
@@ -69,10 +72,10 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
     @classmethod
     def getSiteExampleURLs(self):
         #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
-        return "http://www.literotica.com/s/story-title"
+        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title"
 
     def getSiteURLPattern(self):
-        return r"http://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+        return r"https?://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -97,20 +100,24 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         # author
         a = soup1.find("span", "b-story-user-y")
         self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'])
-        self.story.setMetadata('authorUrl', a.a['href'])
+        authorurl = a.a['href']
+        if authorurl.startswith('//'):
+            authorurl = self.parsedUrl.scheme+':'+authorurl
+        self.story.setMetadata('authorUrl', authorurl)
         self.story.setMetadata('author', a.text)
 
         # get the author page
         try:
-            dataAuth = self._fetchUrl(a.a['href'])
+            dataAuth = self._fetchUrl(authorurl)
             soupAuth = bs.BeautifulSoup(dataAuth)
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(a.a['href'])
+                raise exceptions.StoryDoesNotExist(authorurl)
             else:
                 raise e
 
-        storyLink = soupAuth.find('a', href=url1)
+        ## site has started using //domain.name/asdf urls remove https?: from front
+        storyLink = soupAuth.find('a', href=url1[url1.index(':')+1:])
 
         if storyLink is not None:
             # pull the published date from the author page
@@ -166,7 +173,10 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
             while row['class'] == 'sl':
                 # pages include full URLs.
-                self.chapterUrls.append((row.a.string,row.a['href']))
+                chapurl = row.a['href']
+                if chapurl.startswith('//'):
+                    chapurl = self.parsedUrl.scheme+':'+chapurl
+                self.chapterUrls.append((row.a.string,chapurl))
                 if not row.nextSibling:
                     break
                 row = row.nextSibling

fanficdownloader/adapters/adapter_spikeluvercom.py

@@ -0,0 +1,186 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return SpikeluverComAdapter
+
+
+class SpikeluverComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'slc'
+    SITE_DOMAIN = 'spikeluver.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/SpuffyRealm/'
+    LOGIN_URL = BASE_URL + 'user.php?action=login'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d'
+    METADATA_URL_SUFFIX = '&index=1'
+    AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=5'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception, parameters=None):
+        try:
+            data = self._fetchUrl(url, parameters)
+        except urllib2.HTTPError:
+            raise exception(self.url)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return SpikeluverComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX, exceptions.StoryDoesNotExist)
+
+        # No additional login is required, just check for adult
+        pagetitle_div = soup.find('div', id='pagetitle')
+        if pagetitle_div.a['href'].startswith('javascript:'):
+            if not(self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+        url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
+        soup = self._customized_fetch_url(url, exceptions.StoryDoesNotExist)
+
+        pagetitle_div = soup.find('div', id='pagetitle')
+        self.story.setMetadata('title', pagetitle_div.a.string.strip())
+
+        author_anchor = pagetitle_div.a.findNextSibling('a')
+        url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        components = urlparse.urlparse(url)
+        query_data = urlparse.parse_qs(components.query)
+
+        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', url)
+
+        sort_div = soup.find('div', id='sort')
+        self.story.setMetadata('reviews', sort_div('a')[1].string.strip())
+
+        listbox_tag = soup.find('div', {'class': 'listbox'})
+        for span_tag in listbox_tag('span'):
+            key = span_tag.string.strip(' :')
+            try:
+                value = span_tag.nextSibling.string.strip()
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                # For some reason span_tag.findNextSiblings() only returns tags
+                # instead both NavigableString objects _and_ Tag objects as
+                # documented, so navigate manually
+                sibling = span_tag.nextSibling
+                while sibling:
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        # Encountered next label, break. Not as bad as other
+                        # e-fiction sites, let's hope this is enough for proper
+                        # parsing.
+                        if sibling.name == 'span' and sibling.get('class', None) == 'label':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(self.url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+                    sibling = sibling.nextSibling
+
+                # Remove the preceding break line tag and other crud
+                contents.pop()
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Rated':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Categories':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+
+                    self.story.addToList('category', sibling.string.strip())
+
+            # Seems to be always "None" for some reason
+            elif key == 'Characters':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('characters', sibling.string.strip())
+
+            elif key == 'Genres':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+
+                    self.story.addToList('genre', sibling.string.strip())
+
+            elif key == 'Warnings':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('warnings', sibling.string.strip())
+
+            # Challenges
+
+            elif key == 'Series':
+                a = span_tag.findNextSibling('a')
+                if not a:
+                    continue
+                self.story.setMetadata('series', a.string.strip())
+                self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Word count':
+                self.story.setMetadata('numWords', value)
+
+            elif key == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+        for p_tag in listbox_tag.findNextSiblings('p'):
+            chapter_anchor = p_tag.find('a')
+            title = chapter_anchor.string.strip()
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((title, url))
+
+    def getChapterText(self, url):
+        url += self.AGE_CONSENT_URL_SUFFIX
+        soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
+        return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -66,7 +66,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)"
+        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)?"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -171,7 +171,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:
-                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                if lc2.find('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId'))):
                     i=1
                     break
                 if a[len(a)-1] == lc2:

fanficdownloader/adapters/adapter_voracity2eficcom.py

@@ -0,0 +1,222 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return Voracity2EficComAdapter
+
+
+class Voracity2EficComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'voe'
+    SITE_DOMAIN = 'voracity2.e-fic.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    LOGIN_URL = BASE_URL + 'user.php?action=login'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d'
+    METADATA_URL_SUFFIX = '&index=1'
+    AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=4'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    REQUIRED_SKIN = 'Simple Elegance'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+        self.is_logged_in = False
+
+    def _login(self):
+        # Apparently self.password is only set when login fails, i.e.
+        # the FailedToLogin exception is raised, so the adapter gets new
+        # login data and tries again
+        if self.password:
+            password = self.password
+            username = self.username
+        else:
+            username = self.getConfig('username')
+            password = self.getConfig('password')
+
+        parameters = {
+            'penname': username,
+            'password': password,
+            'submit': 'Submit'}
+
+        class CustomizedFailedToLogin(exceptions.FailedToLogin):
+            def __init__(self, url, passwdonly=False):
+                # Use username variable from outer scope
+                exceptions.FailedToLogin.__init__(self, url, username, passwdonly)
+
+        soup = self._customized_fetch_url(self.LOGIN_URL, CustomizedFailedToLogin, parameters)
+        div = soup.find('div', id='useropts')
+        if not div:
+            raise CustomizedFailedToLogin(self.LOGIN_URL)
+
+        self.is_logged_in = True
+
+    def _customized_fetch_url(self, url, exception, parameters=None):
+        try:
+            data = self._fetchUrl(url, parameters)
+        except urllib2.HTTPError:
+            raise exception(self.url)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return Voracity2EficComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX, urllib2.URLError)
+
+        # Check if the story is for "Registered Users Only", i.e. has adult
+        # content. Based on the "is_adult" attributes either login or raise an
+        # error.
+        errortext_div = soup.find('div', {'class': 'errortext'})
+        if errortext_div:
+            error_text = ''.join(errortext_div(text=True)).strip()
+            if error_text == 'Registered Users Only':
+                if not (self.is_adult or self.getConfig('is_adult')):
+                    raise exceptions.AdultCheckRequired(self.url)
+                self._login()
+            else:
+                # This case usually occurs when the story doesn't exist, but
+                # might potentially be something else, so just raise
+                # FailedToDownload exception with the found error text.
+                raise exceptions.FailedToDownload(error_text)
+
+        url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
+        soup = self._customized_fetch_url(url, urllib2.URLError)
+
+        # If logged in and the skin doesn't match the required skin throw an
+        # error
+        if self.is_logged_in:
+            skin = soup.find('select', {'name': 'skin'}).find('option', selected=True)['value']
+            if skin != self.REQUIRED_SKIN:
+                raise exceptions.FailedToDownload('Required skin "%s" must be set in preferences' % self.REQUIRED_SKIN)
+
+        pagetitle_div = soup.find('div', id='pagetitle')
+        self.story.setMetadata('title', pagetitle_div.a.string)
+
+        author_anchor = pagetitle_div.a.findNextSibling('a')
+        url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        components = urlparse.urlparse(url)
+        query_data = urlparse.parse_qs(components.query)
+
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', url)
+
+        sort_div = soup.find('div', id='sort')
+        self.story.setMetadata('reviews', sort_div('a')[1].string)
+
+        for b_tag in soup.find('div', {'class': 'listbox'})('b'):
+            key = b_tag.string.strip(' :')
+            try:
+                value = b_tag.nextSibling.string.strip()
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                # For some reason span_tag.findNextSiblings() only returns tags
+                # instead both NavigableString objects _and_ Tag objects as
+                # documented, so navigate manually
+                sibling = b_tag.nextSibling
+                while sibling:
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        # Encountered next label, break. This method is the
+                        # safest and most reliable I could think of. Blame
+                        # e-fiction sites that allow their users to include
+                        # arbitrary markup into their summaries and the
+                        # horrible HTML markup.
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(self.url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+                    sibling = sibling.nextSibling
+
+                # Remove the preceding break line tag and other crud
+                contents.pop()
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('category', sibling.string)
+
+            # Seems to be always "None" for some reason
+            elif key == 'Characters':
+                for sibling in b_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('characters', sibling.string)
+
+            elif key == 'Series':
+                a = b_tag.findNextSibling('a')
+                if not a:
+                    continue
+                self.story.setMetadata('series', a.string)
+                self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
+
+            elif key == 'Chapter':
+                self.story.setMetadata('numChapters', int(value))
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Words':
+                self.story.setMetadata('numWords', value)
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value)
+
+            elif key == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+        for b_tag in soup.find('div', id='output').findNextSiblings('b'):
+            chapter_anchor = b_tag.a
+            title = chapter_anchor.string
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((title, url))
+
+    def getChapterText(self, url):
+        url += self.AGE_CONSENT_URL_SUFFIX
+        soup = self._customized_fetch_url(url, urllib2.URLError)
+        return self.utf8FromSoup(url, soup.find('div', id='story'))

index.html

@@ -46,13 +46,6 @@
       {{yourfile}}
       <!--			</div> -->
 
-          <h3>fanfiction.net / fimfiction.net</h3>
-          <p>
-            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
-            are working again.  I'd ask that users limit the number of
-            stories they download from those sites, thanks.
-          </p>
-	  
       {% if authorized %}
       <form action="/fdown" method="post">
 	<div id='urlbox'>
@@ -64,10 +57,9 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Allow https URLs for fimfiction.net.</li>
-              <li>Both allow https URLs for squidge.org/peja and change canonical URLs for squidge.org/peja to https.</li>
-              <li>Fix for some stories' summaries on onedirectionfanfiction.com.</li>
-              <li>Add include/exclude metadata feature.</li>
+              <li>New site: spikeluver.com -- Thanks, cryzed!</li>
+              <li>New site: bloodshedverse.com -- Thanks, cryzed!</li>
+              <li>Updates for site: voracity2.e-fic.com -- Thanks, cryzed!</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -78,7 +70,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-99.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-5-02.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -289,6 +289,10 @@ chapter_title_add_pattern:${index}. ${title}
 ## anthologies.
 anthology_title_pattern:${title} Anthology
 
+## Add tag(s) for anthology (series) books.  Set to empty to not add
+## any anthology tags.
+anthology_tags:Anthology
+
 ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
 ## on '/', so use replace_metadata to change separator first if
 ## needed. Something like: ships=>[ ]*(/|&|&)[ ]*=>/ You can use
@@ -595,6 +599,29 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[bloodshedverse.com]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:warnings,reviews
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
+## Strips links found in the story text
+## Specific to bloodshedverse.com
+strip_text_links:true
+
 [bloodties-fans.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -804,6 +831,14 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fictionpad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1050,6 +1085,16 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[spikeluver.com]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:warnings,reviews
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [stargate-atlantis.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: Atlantis
@@ -1148,6 +1193,13 @@ awards_label:Awards
 
 cover_exclusion_regexp:art/.*Awards.jpg
 
+[voracity2.e-fic.com]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,readings
+reviews_label:Reviews
+readings_label:Readings
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1306,14 +1358,6 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[ficwad.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
 [www.fimfiction.net]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,

status.html

@@ -51,12 +51,6 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-          <h3>fanfiction.net / fimfiction.net</h3>
-          <p>
-            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
-            are working again.  I'd ask that users limit the number of
-            stories they download from those sites, thanks.
-          </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}