Update CLI download zip.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-5-04
+version: 4-5-05
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 23)
+    version             = (1, 8, 24)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -1985,11 +1985,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             for (k,v) in b['all_metadata'].iteritems():
                 #print("merge_meta_books v:%s k:%s"%(v,k))
                 if k in ('numChapters','numWords'):
-                    if k not in book['all_metadata']:
-                        book['all_metadata'][k] = b['all_metadata'][k]
-                    else:
-                        # lot of work for a simple add.
-                        book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
+                    if k in b['all_metadata'] and b['all_metadata'][k]:
+                        if k not in book['all_metadata']:
+                            book['all_metadata'][k] = b['all_metadata'][k]
+                        else:
+                            # lot of work for a simple add.
+                            book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
                 elif k in ('dateUpdated','datePublished','dateCreated',
                            'series','status','title'):
                     pass # handled above, below or skip these for now, not going to do anything with them.

defaults.ini

@@ -790,6 +790,38 @@ extracategories:Harry Potter
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[fanfiction.csodaidok.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-2,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,challenge
+reviews_label:Reviews
+challenge_label:Challenge
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+[fanfic.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.mugglenet.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1004,6 +1036,17 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[nocturnal-light.net]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:readings,reviews
+readings_label:Readings
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [occlumency.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

fanficdownloader/adapters/__init__.py

@@ -126,6 +126,9 @@ import adapter_literotica
 import adapter_voracity2eficcom
 import adapter_spikeluvercom
 import adapter_bloodshedversecom
+import adapter_nocturnallightnet
+import adapter_fanfichu
+import adapter_fanfictioncsodaidokhu
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -39,11 +39,16 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
         self._setURL(self.READ_URL_TEMPLATE % story_no)
         self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
 
-    def _customized_fetch_url(self, url, exception, parameters=None):
-        try:
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
             data = self._fetchUrl(url, parameters)
-        except urllib2.HTTPError:
-            raise exception(self.url)
 
         return BeautifulSoup.BeautifulSoup(data)
 
@@ -64,9 +69,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
         return url
 
     def extractChapterUrlsAndMetadata(self):
-        # Use FailedToDownload exception because that's the only error that can
-        # occur here, missing stories don't return a 404 header
-        soup = self._customized_fetch_url(self.url, exceptions.FailedToDownload)
+        soup = self._customized_fetch_url(self.url)
 
         # Since no 404 error code we have to raise the exception ourselves.
         # A title that is just 'by' indicates that there is no author name
@@ -82,7 +85,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
         # Get the URL to the author's page and find the correct story entry to
         # scrape the metadata
         author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
-        soup = self._customized_fetch_url(author_url, exceptions.FailedToDownload)
+        soup = self._customized_fetch_url(author_url)
 
         story_no = self.story.getMetadata('storyId')
         # Ignore first list_box div, it only contains the author information
@@ -94,6 +97,8 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
             # parse the element
             if query_data['no'] == story_no:
                 break
+        else:
+            raise exceptions.FailedToDownload(self.url)
 
         title_anchor = list_box.find('a', {'class': 'fictitle'})
         self.story.setMetadata('title', title_anchor.string.strip())
@@ -176,7 +181,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
             raise exceptions.AdultCheckRequired(self.url)
 
     def getChapterText(self, url):
-        soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
+        soup = self._customized_fetch_url(url)
         storytext_div = soup.find('div', {'class': 'storytext'})
 
         if self.getConfig('strip_text_links'):

fanficdownloader/adapters/adapter_fanfichu.py

@@ -0,0 +1,185 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanficHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class FanficHuAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'ffh'
+    SITE_DOMAIN = 'fanfic.hu'
+    SITE_LANGUAGE = 'Hungarian'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/merengo/'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%s'
+
+    DATE_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanficHuAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&i=1')
+
+        if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        chapter_options = soup.find('form', action='viewstory.php').select('option')
+        # Remove redundant "Fejezetek" option
+        chapter_options.pop(0)
+
+        # If there is still more than one entry remove chapter overview entry
+        if len(chapter_options) > 1:
+            chapter_options.pop(0)
+
+        for option in chapter_options:
+            url = urlparse.urljoin(self.url, option['value'])
+            self.chapterUrls.append((option.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_id = self.story.getMetadata('storyId')
+        for table in soup('table', {'class': 'mainnav'}):
+            title_anchor = table.find('span', {'class': 'storytitle'}).a
+            href = title_anchor['href']
+            if href.startswith('javascript:'):
+                href = href.rsplit(' ', 1)[1].strip("'")
+            query_data = _get_query_data(href)
+
+            if query_data['sid'] == story_id:
+                break
+        else:
+            # This should never happen, the story must be found on the author's
+            # page.
+            raise exceptions.FailedToDownload(self.url)
+
+        self.story.setMetadata('title', title_anchor.string)
+
+        rows = table('tr')
+
+        anchors = rows[0].div('a')
+        author_anchor = anchors[1]
+        query_data = _get_query_data(author_anchor['href'])
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.BASE_URL, author_anchor['href']))
+        self.story.setMetadata('reviews', anchors[3].string)
+
+        if self.getConfig('keep_summary_html'):
+            self.story.setMetadata('description', self.utf8FromSoup(author_url, rows[1].td))
+        else:
+            self.story.setMetadata('description', ''.join(rows[1].td(text=True)))
+
+        for row in rows[3:]:
+            index = 0
+            cells = row('td')
+
+            while index < len(cells):
+                cell = cells[index]
+                key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
+                try:
+                    value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
+                except AttributeError:
+                    value = None
+
+                if key == 'Kategória':
+                    for anchor in cells[index+1]('a'):
+                        self.story.addToList('category', anchor.string)
+
+                elif key == 'Szereplõk':
+                    if cells[index+1].string:
+                        for name in cells[index+1].string.split(', '):
+                            self.story.addToList('character', name)
+
+                elif key == 'Korhatár':
+                    if value != 'nem korhatáros':
+                        self.story.setMetadata('rating', value)
+
+                elif key == 'Figyelmeztetések':
+                    for b_tag in cells[index+1]('b'):
+                        self.story.addToList('warnings', b_tag.string)
+
+                elif key == 'Jellemzõk':
+                    for genre in cells[index+1].string.split(', '):
+                        self.story.addToList('genre', genre)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Megjelenés':
+                    self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Frissítés':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Szavak':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                index += 2
+
+        if self.story.getMetadata('rating') == '18':
+            if not (self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        story_cell = soup.find('form', action='viewstory.php').parent.parent
+
+        for div in story_cell('div'):
+            div.extract()
+
+        return self.utf8FromSoup(url, story_cell)

fanficdownloader/adapters/adapter_fanfictioncsodaidokhu.py

@@ -0,0 +1,218 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanfictionCsodaidokHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FanfictionCsodaidokHuAdapter(BaseSiteAdapter):
+    _SITE_DOMAIN = 'fanfiction.csodaidok.hu'
+    _BASE_URL = 'http://' + _SITE_DOMAIN + '/'
+    _VIEW_STORY_URL_TEMPLATE = _BASE_URL + 'viewstory.php?sid=%s'
+    _VIEW_CHAPTER_URL_TEMPLATE = _VIEW_STORY_URL_TEMPLATE + '&chapter=%s'
+
+    _STORY_DOES_NOT_EXIST_PAGE_TITLE = 'Cím:  Szerző:'
+    _DATE_FORMAT = '%Y.%m.%d'
+    _SITE_LANGUAGE = 'Hungarian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self._VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self._SITE_DOMAIN)
+        self.story.setMetadata('language', self._SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanfictionCsodaidokHuAdapter._SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls._VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self._VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&chapter=1')
+
+        element = soup.find('div', id='pagetitle')
+        page_title = ''.join(element(text=True)).encode(_SOURCE_CODE_ENCODING)
+        if page_title == self._STORY_DOES_NOT_EXIST_PAGE_TITLE:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        author_url = urlparse.urljoin(self.url, element.a['href'])
+
+        story_id = self.story.getMetadata('storyId')
+        element = soup.find('select', {'name': 'chapter'})
+        if element:
+            for option in element('option'):
+                title = option.string
+                url = self._VIEW_CHAPTER_URL_TEMPLATE % (story_id, option['value'])
+                self.chapterUrls.append((title, url))
+
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+
+        for listbox_div in soup('div', {'class': lambda klass: klass and 'listbox' in klass}):
+            a = listbox_div.div.a
+            if not a['href'].startswith('viewstory.php?sid='):
+                continue
+
+            query_data = _get_query_data(a['href'])
+            if query_data['sid'] == story_id:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = ''.join(a(text=True))
+        self.story.setMetadata('title', title)
+        if not self.chapterUrls:
+            self.chapterUrls.append((title, self.url))
+
+        element = a.findNextSibling('a')
+        self.story.setMetadata('author', element.string)
+        query_data = _get_query_data(element['href'])
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', author_url)
+
+        element = element.findNextSibling('span')
+        rating = element.nextSibling.strip(' [')
+
+        if rating.encode(_SOURCE_CODE_ENCODING) != 'Korhatár nélkül':
+            self.story.setMetadata('rating', rating)
+
+        if rating == '18':
+            raise exceptions.AdultCheckRequired(self.url)
+
+        element = element.findNextSiblings('a')[1]
+        self.story.setMetadata('reviews', element.string)
+
+        sections = listbox_div('div', {'class': lambda klass: klass and klass in ['content', 'tail']})
+        for section in sections:
+            for element in section('span', {'class': 'classification'}):
+                key = element.string.encode(_SOURCE_CODE_ENCODING).strip(' :')
+                try:
+                    value = element.nextSibling.string.encode(_SOURCE_CODE_ENCODING).strip()
+                except AttributeError:
+                    value = None
+
+                if key == 'Tartalom':
+                    contents = []
+                    keep_summary_html = self.getConfig('keep_summary_html')
+
+                    for sibling in _yield_next_siblings(element):
+                        if isinstance(sibling, BeautifulSoup.Tag):
+                            if sibling.name == 'span' and sibling.get('class', None) == 'classification':
+                                break
+
+                            if keep_summary_html:
+                                contents.append(self.utf8FromSoup(author_url, sibling))
+                            else:
+                                contents.append(''.join(sibling(text=True)))
+                        else:
+                            contents.append(sibling)
+                    self.story.setMetadata('description', ''.join(contents))
+
+                elif key == 'Kategória':
+                    for sibling in element.findNextSiblings(['a', 'span']):
+                        if sibling.name == 'span':
+                            break
+
+                        self.story.addToList('category', sibling.string)
+
+                elif key == 'Szereplők':
+                    for name in value.split(', '):
+                        self.story.addToList('characters', name)
+
+                elif key == 'Műfaj':
+                    if value != 'Nincs':
+                        self.story.setMetadata('genre', value)
+
+                elif key == 'Figyelmeztetés':
+                    if value != 'Nincs':
+                        for warning in value.split(', '):
+                            self.story.addToList('warnings', warning)
+
+                elif key == 'Kihívás':
+                    if value != 'Nincs':
+                        self.story.setMetadata('challenge', value)
+
+                elif key == 'Sorozat':
+                    if value != 'Nincs':
+                        self.story.setMetadata('series', value)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                elif key == 'Szavak száma':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Feltöltve':
+                    self.story.setMetadata('datePublished', makeDate(value, self._DATE_FORMAT))
+
+                elif key == 'Frissítve':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self._DATE_FORMAT))
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        contents = []
+
+        notes_div = soup.find('div', id='notes')
+        if notes_div:
+            contents.append(self.utf8FromSoup(url, notes_div))
+            story_div = notes_div.findNextSibling('div')
+        else:
+            element = soup.find('div', {'class': 'jumpmenu'})
+            story_div = element.findNextSibling('div')
+
+        contents.append(self.utf8FromSoup(url, story_div.span))
+        return ''.join(contents)

fanficdownloader/adapters/adapter_nocturnallightnet.py

@@ -0,0 +1,177 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return NocturnalLightNetAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class NocturnalLightNetAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'nln'
+    SITE_DOMAIN = 'nocturnal-light.net'
+    BASE_URL = 'http://' + SITE_DOMAIN + '/fanfiction/'
+    STORY_URL_TEMPLATE = BASE_URL + 'story/%s'
+    AUTHORS_URL_TEMPLATE = BASE_URL + 'authors/%s'
+
+    DATETIME_FORMAT = '%m-%d-%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        url_tokens = self.parsedUrl.path.split('/')
+        story_id = url_tokens[url_tokens.index('story') + 1]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return NocturnalLightNetAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.STORY_URL_TEMPLATE[:-2]) + r'\d+.*$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        # "storycontent" is found in a single-chapter story
+        author_anchor = soup.find('div', id=lambda id: id in ('main', 'storycontent')).h1.a
+        self.story.setMetadata('author', author_anchor.string)
+
+        url_tokens = author_anchor['href'].split('/')
+        author_id = url_tokens[url_tokens.index('authors')+1]
+        self.story.setMetadata('authorId', author_id)
+        self.story.setMetadata('authorUrl', self.AUTHORS_URL_TEMPLATE % author_id)
+
+        chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/'))
+        for chapter_anchor in chapter_anchors:
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((chapter_anchor.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+        for listbox in soup('div', {'class': 'listbox'}):
+            url_tokens = listbox.a['href'].split('/')
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if story_id == url_tokens[url_tokens.index('story')+1]:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = listbox.a.string
+        self.story.setMetadata('title', title)
+
+        # No chapter anchors found in the original story URL, so the story has
+        # only a single chapter.
+        if not chapter_anchors:
+            self.chapterUrls.append((title, self.url))
+
+        for b_tag in listbox('b'):
+            key = b_tag.string.strip(':')
+            try:
+                value = b_tag.nextSibling.string.replace('•', '').strip(': ')
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(b_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(author_url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Pop last break line tag
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'b']):
+                    if sibling.name == 'b':
+                        break
+
+                    self.story.addToList('category', sibling.string)
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+                # Also parse reviews number which lies right after the chapters
+                # section
+                reviews_anchor = b_tag.findNextSibling('a')
+                reviews = reviews_anchor.string.split(' ')[1].strip('()')
+                self.story.setMetadata('reviews', reviews)
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Date Added':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Last Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value.split()[0])
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='storytext'))

fanficdownloader/adapters/adapter_spikeluvercom.py

@@ -12,6 +12,16 @@ def getClass():
     return SpikeluverComAdapter
 
 
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
 class SpikeluverComAdapter(BaseSiteAdapter):
     SITE_ABBREVIATION = 'slc'
     SITE_DOMAIN = 'spikeluver.com'
@@ -23,6 +33,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
     AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=5'
 
     DATETIME_FORMAT = '%m/%d/%Y'
+    STORY_DOES_NOT_EXIST_ERROR_TEXT = 'That story does not exist on this archive.  You may search for it or return to the home page.'
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -34,11 +45,16 @@ class SpikeluverComAdapter(BaseSiteAdapter):
         self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
         self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
 
-    def _customized_fetch_url(self, url, exception, parameters=None):
-        try:
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
             data = self._fetchUrl(url, parameters)
-        except urllib2.HTTPError:
-            raise exception(self.url)
 
         return BeautifulSoup.BeautifulSoup(data)
 
@@ -54,7 +70,13 @@ class SpikeluverComAdapter(BaseSiteAdapter):
         return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
 
     def extractChapterUrlsAndMetadata(self):
-        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX, exceptions.StoryDoesNotExist)
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX)
+
+        errortext_div = soup.find('div', {'class': 'errortext'})
+        if errortext_div:
+            error_text = ''.join(errortext_div(text=True)).strip()
+            if error_text == self.STORY_DOES_NOT_EXIST_ERROR_TEXT:
+                raise exceptions.StoryDoesNotExist(self.url)
 
         # No additional login is required, just check for adult
         pagetitle_div = soup.find('div', id='pagetitle')
@@ -63,7 +85,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                 raise exceptions.AdultCheckRequired(self.url)
 
         url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
-        soup = self._customized_fetch_url(url, exceptions.StoryDoesNotExist)
+        soup = self._customized_fetch_url(url)
 
         pagetitle_div = soup.find('div', id='pagetitle')
         self.story.setMetadata('title', pagetitle_div.a.string.strip())
@@ -95,11 +117,7 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                 contents = []
                 keep_summary_html = self.getConfig('keep_summary_html')
 
-                # For some reason span_tag.findNextSiblings() only returns tags
-                # instead both NavigableString objects _and_ Tag objects as
-                # documented, so navigate manually
-                sibling = span_tag.nextSibling
-                while sibling:
+                for sibling in _yield_next_siblings(span_tag):
                     if isinstance(sibling, BeautifulSoup.Tag):
                         # Encountered next label, break. Not as bad as other
                         # e-fiction sites, let's hope this is enough for proper
@@ -113,7 +131,6 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                             contents.append(''.join(sibling(text=True)))
                     else:
                         contents.append(sibling)
-                    sibling = sibling.nextSibling
 
                 # Remove the preceding break line tag and other crud
                 contents.pop()
@@ -175,12 +192,15 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
 
         for p_tag in listbox_tag.findNextSiblings('p'):
-            chapter_anchor = p_tag.find('a')
+            chapter_anchor = p_tag.find('a', href=lambda href: href and href.startswith('viewstory.php?sid='))
+            if not chapter_anchor:
+                continue
+
             title = chapter_anchor.string.strip()
             url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
             self.chapterUrls.append((title, url))
 
     def getChapterText(self, url):
         url += self.AGE_CONSENT_URL_SUFFIX
-        soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
+        soup = self._customized_fetch_url(url)
         return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/adapter_voracity2eficcom.py

@@ -12,6 +12,16 @@ def getClass():
     return Voracity2EficComAdapter
 
 
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
 class Voracity2EficComAdapter(BaseSiteAdapter):
     SITE_ABBREVIATION = 'voe'
     SITE_DOMAIN = 'voracity2.e-fic.com'
@@ -65,11 +75,16 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
 
         self.is_logged_in = True
 
-    def _customized_fetch_url(self, url, exception, parameters=None):
-        try:
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
             data = self._fetchUrl(url, parameters)
-        except urllib2.HTTPError:
-            raise exception(self.url)
 
         return BeautifulSoup.BeautifulSoup(data)
 
@@ -85,7 +100,7 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
         return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
 
     def extractChapterUrlsAndMetadata(self):
-        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX, urllib2.URLError)
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX)
 
         # Check if the story is for "Registered Users Only", i.e. has adult
         # content. Based on the "is_adult" attributes either login or raise an
@@ -104,7 +119,7 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
                 raise exceptions.FailedToDownload(error_text)
 
         url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
-        soup = self._customized_fetch_url(url, urllib2.URLError)
+        soup = self._customized_fetch_url(url)
 
         # If logged in and the skin doesn't match the required skin throw an
         # error
@@ -142,11 +157,7 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
                 contents = []
                 keep_summary_html = self.getConfig('keep_summary_html')
 
-                # For some reason span_tag.findNextSiblings() only returns tags
-                # instead both NavigableString objects _and_ Tag objects as
-                # documented, so navigate manually
-                sibling = b_tag.nextSibling
-                while sibling:
+                for sibling in _yield_next_siblings(b_tag):
                     if isinstance(sibling, BeautifulSoup.Tag):
                         # Encountered next label, break. This method is the
                         # safest and most reliable I could think of. Blame
@@ -162,7 +173,6 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
                             contents.append(''.join(sibling(text=True)))
                     else:
                         contents.append(sibling)
-                    sibling = sibling.nextSibling
 
                 # Remove the preceding break line tag and other crud
                 contents.pop()
@@ -218,5 +228,5 @@ class Voracity2EficComAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
         url += self.AGE_CONSENT_URL_SUFFIX
-        soup = self._customized_fetch_url(url, urllib2.URLError)
+        soup = self._customized_fetch_url(url)
         return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/story.py

@@ -28,6 +28,9 @@ import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable
 
+SPACE_REPLACE=u'\s'
+SPLIT_META=u'\,'
+
 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
 
@@ -230,19 +233,19 @@ class InExMatch:
     def  __init__(self,line):
         if "=~" in line:
             (self.keys,self.match) = line.split("=~")
-            self.match = self.match.replace('\s',' ')
+            self.match = self.match.replace(SPACE_REPLACE,' ')
             self.regex = re.compile(self.match)
         elif "!~" in line:
             (self.keys,self.match) = line.split("!~")
-            self.match = self.match.replace('\s',' ')
+            self.match = self.match.replace(SPACE_REPLACE,' ')
             self.regex = re.compile(self.match)
             self.negate = True
         elif "==" in line:
             (self.keys,self.match) = line.split("==")
-            self.match = self.match.replace('\s',' ')
+            self.match = self.match.replace(SPACE_REPLACE,' ')
         elif "!=" in line:
             (self.keys,self.match) = line.split("!=")
-            self.match = self.match.replace('\s',' ')
+            self.match = self.match.replace(SPACE_REPLACE,' ')
             self.negate = True
         self.keys = map( lambda x: x.strip(), self.keys.split(",") )
 
@@ -301,8 +304,7 @@ class Story(Configurable):
         self.logfile=None # cheesy way to carry log file forward across update.
 
         ## Look for config parameter, split and add each to metadata field.
-        for (config,metadata) in [("extratags","extratags"),
-                                  ("extracategories","category"),
+        for (config,metadata) in [("extracategories","category"),
                                   ("extragenres","genre"),
                                   ("extracharacters","characters"),
                                   ("extraships","ships"),
@@ -416,14 +418,13 @@ class Story(Configurable):
                 # A way to explicitly include spaces in the
                 # replacement string.  The .ini parser eats any
                 # trailing spaces.
-                replacement=replacement.replace('\s',' ')
+                replacement=replacement.replace(SPACE_REPLACE,' ')
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
 
     def doReplacements(self,value,key):
         value = self.do_in_ex_clude('include_metadata_pre',value,key)
         value = self.do_in_ex_clude('exclude_metadata_pre',value,key)
-        # if key=='characters' and value=='Bilbo':
-        #     value = ''
+
         for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
@@ -434,7 +435,18 @@ class Story(Configurable):
                     doreplace = condval != None and condregexp.search(condval)
 
                 if doreplace:
-                    value = regexp.sub(replacement,value)
+                    # split into more than one list entry if list and
+                    # SPLIT_META present in replacement string.  Split
+                    # first, then regex sub.
+                    if self.isList(key) and SPLIT_META in replacement:
+                        repllist = replacement.split(SPLIT_META)
+                        for repl in repllist[1:]:
+                            self.addToList(key,regexp.sub(repl,value))
+                        value = regexp.sub(repllist[0],value)
+                    else:            
+                        value = regexp.sub(replacement,value)
+                    
+
         value = self.do_in_ex_clude('include_metadata_post',value,key)
         value = self.do_in_ex_clude('exclude_metadata_post',value,key)
         return value
@@ -451,7 +463,7 @@ class Story(Configurable):
             return value
 
         if self.isList(key):
-            join_string = self.getConfig("join_string_"+key,u", ").replace('\s',' ')
+            join_string = self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ')
             value = join_string.join(self.getList(key, removeallentities, doreplacements=True))
             if doreplacements:
                 value = self.doReplacements(value,key+"_LIST")
@@ -502,7 +514,7 @@ class Story(Configurable):
                     auth=removeAllEntities(auth)
 
                 htmllist.append(linkhtml%('author',aurl,auth))
-            join_string = self.getConfig("join_string_authorHTML",u", ").replace('\s',' ')
+            join_string = self.getConfig("join_string_authorHTML",u", ").replace(SPACE_REPLACE,' ')
             self.setMetadata('authorHTML',join_string.join(htmllist))
         else:
             self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements),
@@ -534,7 +546,7 @@ class Story(Configurable):
                     v=removeAllEntities(v)
 
                 htmllist.append(linkhtml%(k,url,v))
-            join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace('\s',' ')
+            join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace(SPACE_REPLACE,' ')
             self.setMetadata(k+'HTML',join_string.join(htmllist))
 
         for k in self.getValidMetaList():

index.html

@@ -55,11 +55,15 @@
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>Changes:</h3>
+	  <p>
+            Now supporting over 100 different sites!  Thanks, cryzed, for pushing us over the top.
+	  </p>
 	  <p>
 	    <ul>
-              <li>New site: spikeluver.com -- Thanks, cryzed!</li>
-              <li>New site: bloodshedverse.com -- Thanks, cryzed!</li>
-              <li>Updates for site: voracity2.e-fic.com -- Thanks, cryzed!</li>
+              <li>New site: nocturnal-light.net -- Thanks, cryzed!</li>
+              <li>New site: fanfic.hu (Hungarian language) -- Thanks, cryzed!</li>
+              <li>New site: fanfiction.csodaidok.hu (Hungarian language) -- Thanks, cryzed!</li>
+              <li>Improvements &amp; fixes for recently added sites -- Thanks, cryzed!</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -70,7 +74,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-5-02.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-5-03.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -232,6 +232,9 @@ connect_timeout:60.0
 ## Make sure to keep at least one space at the start of each line and
 ## to escape % to %%, if used.
 ## template => regexp to match => GC Setting to use.
+## To use this, make sure you go to the Generate Cover tab in FFDL
+## config and check 'Allow generate_cover_settings from personal.ini
+## to override'
 #generate_cover_settings:
 # ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
 # ${category} => Star Trek => StarTrekCover
@@ -781,6 +784,38 @@ extracategories:Harry Potter
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[fanfiction.csodaidok.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-2,auto
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,challenge
+reviews_label:Reviews
+challenge_label:Challenge
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+[fanfic.hu]
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+website_encodings:ISO-8859-1,auto
+
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.mugglenet.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -995,6 +1030,17 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[nocturnal-light.net]
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:readings,reviews
+readings_label:Readings
+reviews_label:Reviews
+
+## Site dedicated to these categories/characters/ships
+extracharacters:Spike,Buffy
+extracategories:Buffy the Vampire Slayer
+
 [occlumency.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In