Bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-87
+version: 4-4-91
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 04)
+    version             = (1, 8, 10)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -209,6 +209,7 @@ class ConfigWidget(QWidget):
         prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
         prefs['mark'] = self.basic_tab.mark.isChecked()
         prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
+        prefs['autoconvert'] = self.basic_tab.autoconvert.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -422,6 +423,11 @@ class BasicTab(QWidget):
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
+        self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
+        self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
+        self.autoconvert.setChecked(prefs['autoconvert'])
+        self.l.addWidget(self.autoconvert)
+
         gui_gb = groupbox = QGroupBox(_("GUI Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)

calibre-plugin/ffdl_plugin.py

@@ -1232,6 +1232,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             cp_plugin = self.gui.iactions['Count Pages']
             cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
 
+        if prefs['autoconvert']:
+            self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
+            self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
+
     def download_list_completed(self, job, options={},merge=False):
         if job.failed:
             self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -1393,7 +1397,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
             self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
             self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
-            label = custom_columns[prefs['errorcol']]['label']
+            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
+                label = custom_columns[prefs['errorcol']]['label']
+            else:
+                label = None
             LoopProgressDialog(self.gui,
                                book_list,
                                partial(self.update_error_column_loop, db=self.gui.current_db, label=label),
@@ -1402,12 +1409,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                win_title=_("Update calibre for BAD FanFiction stories"),
                                status_prefix=_("Updated"))
 
-    def update_error_column_loop(self,book,db=None,label='errorcol'):
-        if book['calibre_id']:
-            custom_columns = self.gui.library_view.model().custom_columns
-            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
-                logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+    def update_error_column_loop(self,book,db=None,label=None):
+        if book['calibre_id'] and label:
+            logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
 
     def add_book_or_update_format(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -1545,8 +1550,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
 
                     if meta not in book['all_metadata']:
-                        logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
-                        continue
+                        # if double quoted, use as a literal value.
+                        if meta[0] == '"' and meta[-1] == '"':
+                            val = meta[1:-1]
+                            logger.debug("No metadata value for %s, setting custom column(%s) literally to %s."%(meta,custcol,val))
+                        else:
+                            logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                            continue
+                    else:
+                        val = book['all_metadata'][meta]    
                     
                     if custcol not in custom_columns:
                         continue
@@ -1558,11 +1570,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
                             if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
                                 # re-split list, strip commas, convert to floats, sum up.
-                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                                val = sum([ float(x.replace(",","")) for x in val.split(", ") ])
                             else:
-                                val = unicode(book['all_metadata'][meta]).replace(",","")
+                                val = unicode(val).replace(",","")
                         else:
-                            val = book['all_metadata'][meta]
+                            val = val
                         if val != '':
                             db.set_custom(book_id, val, label=label, commit=False)
 
@@ -1577,8 +1589,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         except:
                             pass
 
-                        if book['all_metadata'][meta]:
-                            vallist = [book['all_metadata'][meta]]
+                        if val:
+                            vallist = [val]
                             
                         db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
 

calibre-plugin/prefs.py

@@ -34,6 +34,7 @@ default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
 default_prefs['mark'] = False
 default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'

defaults.ini

@@ -817,12 +817,24 @@ dislikes_label:Dislikes
 ## Clear FanFiction from defaults, site is original fiction.
 extratags:
 
-extra_valid_entries:size,universe,codes
-#extra_titlepage_entries:size,universe,codes
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice
 
 size_label:Size
 universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
 codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true
 
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The

fanficdownloader/adapters/__init__.py

@@ -122,6 +122,7 @@ import adapter_scarheadnet
 import adapter_fictionpadcom
 import adapter_storiesonlinenet
 import adapter_trekiverseorg
+import adapter_literotica
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -77,7 +77,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archiveofourown.org'
 
-    @classmethod
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
 
@@ -188,7 +188,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('numChapters',len(chapters))
         logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
         if len(chapters)==1:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
         else:
             for index, chapter in enumerate(chapters):
                 # strip just in case there's tags, like <i> in chapter titles.

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -72,6 +72,13 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
 
+    def _fetchUrl(self,url):
+        time.sleep(1.0) ## ffnet(and, I assume, fpcom) tends to fail
+                        ## more if hit too fast.  This is in
+                        ## additional to what ever the
+                        ## slow_down_sleep_time setting is.
+        return BaseSiteAdapter._fetchUrl(self,url)
+
     def extractChapterUrlsAndMetadata(self):
 
         # fetch the chapter.  From that we will get almost all the
@@ -157,7 +164,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 
                 
             
-        a = soup.find('a', href='http://www.fictionratings.com/')
+        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
         rating = a.string
         if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
             rating = rating[8:]
@@ -275,7 +282,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(4.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,11 +37,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -51,7 +46,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         
-        
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
         

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -128,23 +128,22 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # <span class="label">Rated:</span> NC-17<br /> etc
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
-            value = labelspan.nextSibling
+            val = labelspan.nextSibling
+            value = unicode('')
+            while val and not defaultGetattr(val,'class') == 'label':
+                value += unicode(val)
+                val = val.nextSibling
             label = labelspan.string
+            #print("label:%s\nvalue:%s"%(label,value))
 
             if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,value)
 
             if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+                self.story.setMetadata('rating', stripHTML(value))
 
             if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+                self.story.setMetadata('numWords', stripHTML(value))
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -167,7 +166,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
-                if 'Yes' in value:
+                if 'Yes' in stripHTML(value):
                     self.story.setMetadata('status', 'Completed')
                 else:
                     self.story.setMetadata('status', 'In-Progress')

fanficdownloader/adapters/adapter_literotica.py

@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+class LiteroticaSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.story.setMetadata('siteabbrev','litero')
+
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        
+        self.origurl = url
+        if "http://www.i." in self.origurl:
+            ## accept m(mobile)url, but use www.
+            self.origurl = self.origurl.replace("http://www.i.","http://www.")
+
+        # normalized story URL.
+        self._setURL("http://"+self.getSiteDomain()\
+                         +"/s/"+self.story.getMetadata('storyId'))
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%m/%d/%y'
+        
+    @staticmethod
+    def getSiteDomain():
+        return 'www.literotica.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.literotica.com', 'www.i.literotica.com']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
+        return "http://www.literotica.com/s/story-title"
+
+    def getSiteURLPattern(self):
+        return r"http://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        url1 = self.origurl
+        logger.debug("first page URL: "+url1)
+
+        try:
+            data1 = self._fetchUrl(url1)
+            soup1 = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url1)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # author
+        a = soup1.find("span", "b-story-user-y")
+        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'])
+        self.story.setMetadata('authorUrl', a.a['href'])
+        self.story.setMetadata('author', a.text)
+
+        # get the author page
+        try:
+            dataAuth = self._fetchUrl(a.a['href'])
+            soupAuth = bs.BeautifulSoup(dataAuth)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(a.a['href'])
+            else:
+                raise e
+
+        storyLink = soupAuth.find('a', href=url1)
+
+        if storyLink is not None:
+            # pull the published date from the author page
+            # default values from single link.  Updated below if multiple chapter.
+            date = storyLink.parent.parent.findAll('td')[-1].text
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
+
+        # find num of pages
+        # find a "3 Pages:" string on the page and parse it
+        pgs = soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0]
+
+        # If there are multiple pages, find and request the last page
+        if "1" != pgs:
+            logger.debug("last page number: "+pgs)
+            try:
+                data2 = self._fetchUrl(url1, {'page': pgs})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    # TODO: Probably should reformat this
+                    raise exceptions.StoryDoesNotExist(url1, {'page': pgs})
+                else:
+                    raise e
+        else:
+            #If we're already on the last page, copy the soup
+            soup2 = soup1
+
+        # parse out the list of chapters
+        chaps = soup2.find('div', id='b-series')
+        if chaps:  # may be one post only
+            #self.chapterUrls = [(ch.a.text, ch.a['href']) for ch in chaps.findAll('li')]
+
+            # if there are chapters, lets pull them and title from the
+            # author page because *this* chapter is omitted from the
+            # list on the last page.
+            row = storyLink.parent.parent.previousSibling
+            while row['class'] != 'ser-ttl':
+                row = row.previousSibling
+
+            seriesTitle = stripHTML(row)
+            if seriesTitle:
+                # this regex is deliberately greedy. We want to get the biggest match before a ':'
+                self.story.setMetadata('title', re.match('(.*):[^:]*$', seriesTitle).group(1))
+            else:
+                self.story.setMetadata('title', soup1.h1.string)
+
+            # now chapter list.  Assumed oldest to newest.
+            self.chapterUrls = []
+            row = row.nextSibling
+                
+            self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+            while row['class'] == 'sl':
+                # pages include full URLs.
+                self.chapterUrls.append((row.a.string,row.a['href']))
+                if not row.nextSibling:
+                    break
+                row = row.nextSibling
+
+            row = row.previousSibling
+            self.story.setMetadata('dateUpdated',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+                
+        else:  # if one post only
+            self.chapterUrls = [(soup1.h1.string, url1)]
+            self.story.setMetadata('title', soup1.h1.string)
+
+        # normalize on first chapter URL.
+        self._setURL(self.chapterUrls[0][1])
+
+        # reset storyId to first chapter.
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+
+        self.story.setMetadata('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)
+        self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
+
+        return
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5)
+        data1 = self._fetchUrl(url)
+        soup1 = bs.BeautifulSoup(data1)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        story1 = soup1.find('div', 'b-story-body-x').p
+        story1.name='div'
+        story1.append('<br>')
+        storytext = self.utf8FromSoup(url,story1)
+
+        # find num pages
+        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
+        logger.debug("pages: "+str(pgs))
+
+        # get all the pages
+        for i in xrange(2, pgs+1):
+            try:
+                logger.debug("fetching page "+str(i))
+                time.sleep(0.5)
+                data2 = self._fetchUrl(url, {'page': i})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+                story2 = soup2.find('div', 'b-story-body-x').p
+                story2.name='div'
+                story2.append('<br>')
+                storytext += self.utf8FromSoup(url,story2)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        return storytext
+
+
+def getClass():
+    return LiteroticaSiteAdapter
+
+
+

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -37,11 +37,6 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -147,6 +142,10 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         a = soup.find('h1')
         self.story.setMetadata('title',stripHTML(a))
         
+        notice = soup.find('div', {'class' : 'notice'})
+        if notice:
+            self.story.setMetadata('notice',unicode(notice))
+
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"/a/\w+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -185,23 +184,74 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text)
         
         lc4 = lc2.findNext('td', {'class' : 'lc4'})        
-        
+        desc = lc4.contents[0]
+
         try:
             a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
-            i = a.parent.text.split('(')[1].split(')')[0]
-            self.setSeries(stripHTML(a), i)
-            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
+            if a:
+                # if there's a number after the series name, series_contents is a two element list:
+                # [<a href="...">Title</a>, u' (2)']
+                series_contents = a.parent.contents
+                i = 0 if len(series_contents) == 1 else series_contents[1].strip(' ()')
+                seriesUrl = 'http://'+self.host+a['href']
+                self.story.setMetadata('seriesUrl',seriesUrl)
+                series_name = stripHTML(a)
+                logger.debug("Series name= %s" % series_name)
+                series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl))
+                if series_soup:
+                    logger.debug("Retrieving Series - looking for name")
+                    series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
+                    logger.debug("Series name: '{0}'".format(series_name))
+                self.setSeries(series_name, i)
+                desc = lc4.contents[2]
+                # Check if series is in a universe
+                universes_soup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')  + "&type=uni")) 
+#                logger.debug("Universe page=", universes_soup)
+                if universes_soup:
+                    universes = universes_soup.findAll('div', {'class' : 'ser-box'})
+                    logger.debug("Number of Universes: %d" % len(universes))
+                    for universe in universes:
+                        logger.debug("universe.find('a')={0}".format(universe.find('a')))
+                        # The universe id is in an "a" tag that has an id but nothing else. It is the first tag.
+                        # The id is prefixed with the letter "u".
+                        universe_id = universe.find('a')['id'][1:]
+                        logger.debug("universe_id='%s'" % universe_id)
+                        universe_name = universe.find('div', {'class' : 'ser-name'}).text.partition(' ')[2]
+                        logger.debug("universe_name='%s'" % universe_name)
+                        # If there is link to the story, we have the right universe
+                        story_a = universe.find('a', {'href' : '/s/'+self.story.getMetadata('storyId')})
+                        if story_a:
+                            logger.debug("Story is in a series that is in a universe! The universe is '%s'" % universe_name)
+                            self.story.setMetadata("universe", universe_name)
+                            self.story.setMetadata('universeUrl','http://'+self.host+ '/library/universe.php?id=' + universe_id)
+                            break
         except:
             pass
         try:
             a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
             if a:
                 self.story.setMetadata("universe",stripHTML(a))
+                desc = lc4.contents[2]
+                # Assumed only one universe, but it does have a URL--use universeHTML
+                universe_name = stripHTML(a)
+                universeUrl = 'http://'+self.host+a['href']
+                logger.debug("Retrieving Universe - about to get page")
+                universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl))
+                logger.debug("Retrieving Universe - have page")
+                if universe_soup:
+                    logger.debug("Retrieving Universe - looking for name")
+                    universe_name = universe_soup.find('span', {'id' : 'ptitle'}).text.partition(' &mdash;')[0]
+                    logger.debug("Universes name: '{0}'".format(universe_name))
+
+                self.story.setMetadata('universeUrl',universeUrl)
+                logger.debug("Setting universe name: '{0}'".format(universe_name))
+                self.story.setMetadata('universe',universe_name)
+                if self.getConfig("universe_as_series"):
+                    self.setSeries(universe_name, 0)
+                    self.story.setMetadata('seriesUrl',universeUrl)
         except:
             pass
-            
 
-        desc = lc4.contents[0]
         self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
             
         for b in lc4.findAll('b'):
@@ -255,8 +305,18 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         # some big chapters are split over several pages
         pager = div.find('span', {'class' : 'pager'})
         if pager != None:
+            a = pager.previousSibling
+            while a != None:
+                logger.debug("before pager: {0}".format(a))
+                b = a.previousSibling
+                a.extract()
+                a = b
+
             urls=pager.findAll('a')
             urls=urls[:len(urls)-1]
+            pager.extract()
+            div.contents = div.contents[2:]
+#            logger.debug(div)
                         
             for ur in urls:
                 soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
@@ -264,38 +324,76 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         
                 div1 = soup.find('div', {'id' : 'story'})
                 
-                # appending next section
-                last=div.findAll('p')
-                next=div1.find('span', {'class' : 'conTag'}).nextSibling
-            
-                last[len(last)-1]=last[len(last)-1].append(next)
-                div.append(div1)
-            
-        # removing all the left-over stuff    
-        for a in div.findAll('span'):
-            a.extract() 
-            
-        for a in div.findAll('h1'):
-            a.extract()
-        for a in div.findAll('h2'):
-            a.extract()
-        for a in div.findAll('h3'):
-            a.extract()
-        for a in div.findAll('h4'):
-            a.extract()
-        for a in div.findAll('br'):
-            a.extract()
-        for a in div.findAll('div', {'class' : 'date'}):
-            a.extract()
-            
-        a = div.find('form')
-        if a != None:
-            b = a.nextSibling
-            while b != None:
+                # Find the "Continues" marker on the current page and remove everything after that. 
+                continues = div.find('span', {'class' : 'conTag'})
+                if continues != None:
+                    while continues != None:
+#                        logger.debug("removing end: {0}".format(continues))
+                        b = continues.nextSibling
+                        continues.extract()
+                        continues = b
+
+                # Find the "Continued" marker and delete everything before that
+                continued = div1.find('span', {'class' : 'conTag'})
+                if continued != None:
+                    a = continued.previousSibling
+                    while a != None:
+#                        logger.debug("before conTag: {0}".format(a))
+                        b = a.previousSibling
+                        a.extract()
+                        a = b
+                # Remove the pager from the end if this is the last page
+                endPager = div1.find('span', {'class' : 'pager'})
+                if endPager != None:
+                    b = endPager.nextSibling
+                    while endPager != None:
+                        logger.debug("removing end: {0}".format(endPager))
+                        b = endPager.nextSibling
+                        endPager.extract()
+                        endPager = b
+                    div1.contents = div1.contents[:len(div1) - 2]
+#                logger.debug("after removing pager: {0}".format(div1))
+                for tag in div1.contents[2:]:
+                    div.append(tag)
+
+        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus 
+        # everything before them can be discarded. 
+        postedDates = div.findAll('div', {'class' : 'date'})
+        if postedDates:
+            a = postedDates[0].previousSibling
+            while a != None:
+#                logger.debug("before dates: {0}".format(a))
+                b = a.previousSibling
                 a.extract()
-                a=b
-                b=b.nextSibling
-        
+                a = b
+            for a in div.findAll('div', {'class' : 'date'}):
+                a.extract()
+
+        # For single chapter stories, there is a copyright statement. Remove this and everything
+        # before it.
+        copy = div.find('h4', {'class': 'copy'})
+        while copy != None:
+#            logger.debug("before copyright: {0}".format(copy))
+            b = copy.previousSibling
+            copy.extract()
+            copy = b
+
+        # For a story or the last chapter, remove voting form and the in library box
+        a = div.find('div', {'id' : 'vote-form'})
+        if a != None:
+            a.extract()
+        a = div.find('div', {'id' : 'b-man-div'})
+        if a != None:
+            a.extract()
+
+        # Kill the "The End" header and everything after it.
+        a = div.find(['h2', 'h3'], {'class' : 'end'})
+        logger.debug("Chapter end= '{0}'".format(a))
+        while a != None:
+            b = a.nextSibling
+            a.extract()
+            a=b
+
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/htmlheuristics.py

@@ -137,32 +137,35 @@ def replace_br_with_p(body):
             contentLinesSum += lineLen
             if lineLen > longestLineLength:
                 longestLineLength = lineLen
+                
+    if contentLines == 0:
+        contentLines = 1
 
     averageLineLength = contentLinesSum/contentLines
 
     logger.debug(u'---')
-    logger.debug(u'Lines.............: ' + str(len(lines)))
-    logger.debug(u'contentLines......: ' + str(contentLines))
-    logger.debug(u'contentLinesSum...: ' + str(contentLinesSum))
-    logger.debug(u'longestLineLength.: ' + str(longestLineLength))
-    logger.debug(u'averageLineLength.: ' + str(averageLineLength))
+    logger.debug(u'Lines.............: ' + unicode(len(lines)))
+    logger.debug(u'contentLines......: ' + unicode(contentLines))
+    logger.debug(u'contentLinesSum...: ' + unicode(contentLinesSum))
+    logger.debug(u'longestLineLength.: ' + unicode(longestLineLength))
+    logger.debug(u'averageLineLength.: ' + unicode(averageLineLength))
 
     if breaksMaxIndex == len(breaksCount)-1 and breaksMax < 2:
         breaksMaxIndex = 0
         breaksMax = breaksCount[0]
 
     logger.debug(u'---')
-    logger.debug(u'breaks 1: ' + str(breaksCount[0]))
-    logger.debug(u'breaks 2: ' + str(breaksCount[1]))
-    logger.debug(u'breaks 3: ' + str(breaksCount[2]))
-    logger.debug(u'breaks 4: ' + str(breaksCount[3]))
-    logger.debug(u'breaks 5: ' + str(breaksCount[4]))
-    logger.debug(u'breaks 6: ' + str(breaksCount[5]))
-    logger.debug(u'breaks 7: ' + str(breaksCount[6]))
-    logger.debug(u'breaks 8: ' + str(breaksCount[7]))
+    logger.debug(u'breaks 1: ' + unicode(breaksCount[0]))
+    logger.debug(u'breaks 2: ' + unicode(breaksCount[1]))
+    logger.debug(u'breaks 3: ' + unicode(breaksCount[2]))
+    logger.debug(u'breaks 4: ' + unicode(breaksCount[3]))
+    logger.debug(u'breaks 5: ' + unicode(breaksCount[4]))
+    logger.debug(u'breaks 6: ' + unicode(breaksCount[5]))
+    logger.debug(u'breaks 7: ' + unicode(breaksCount[6]))
+    logger.debug(u'breaks 8: ' + unicode(breaksCount[7]))
     logger.debug(u'----')
-    logger.debug(u'max found: ' + str(breaksMax))
-    logger.debug(u'max Index: ' + str(breaksMaxIndex))
+    logger.debug(u'max found: ' + unicode(breaksMax))
+    logger.debug(u'max Index: ' + unicode(breaksMaxIndex))
     logger.debug(u'----')
 
     if breaksMaxIndex > 0 and breaksCount[0] > breaksMax and averageLineLength < 90:
@@ -173,13 +176,13 @@ def replace_br_with_p(body):
     for i in range(len(breaksCount)):
         # if i > 0 or breaksMaxIndex == 0:
         if i <= breaksMaxIndex:
-            logger.debug(str(i) + u' <= breaksMaxIndex (' + str(breaksMaxIndex) + u')')
+            logger.debug(unicode(i) + u' <= breaksMaxIndex (' + unicode(breaksMaxIndex) + u')')
             body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body)
         elif i == breaksMaxIndex+1:
-            logger.debug(str(i) + u' == breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            logger.debug(unicode(i) + u' == breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')')
             body = breaksRegexp[i].sub(r'\1</p>\n<p><br/></p>\n<p>\3', body)
         else:
-            logger.debug(str(i) + u' > breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            logger.debug(unicode(i) + u' > breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')')
             body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body)
 
     body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body)
@@ -230,7 +233,7 @@ def replace_br_with_p(body):
     return tag_sanitizer(body)
 
 def is_valid_block(block):
-    return str(block).find('<') == 0 and str(block).find('<!') != 0
+    return unicode(block).find('<') == 0 and unicode(block).find('<!') != 0
 
 def soup_up_div(body):
     blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'p', 'pre', 'table', 'ul']
@@ -247,8 +250,8 @@ def soup_up_div(body):
     lastElement = 1 # 1 = block, 2 = nested, 3 = invalid
 
     for i in soup.contents[0]:
-        if str(i).strip().__len__() > 0:
-            s = str(i)
+        if unicode(i).strip().__len__() > 0:
+            s = unicode(i)
             if  type(i) == bs.Tag:
                 if  i.name in blockTags:
                     if lastElement > 1:
@@ -308,7 +311,7 @@ def tag_sanitizer(html):
         is_closed = is_closed_tag(rTag[0]) or is_comment_tag(rTag[0])
 
         # is_comment = is_comment_tag(rTag[0])
-        # logger.debug(u'%s >  isEnd: %s >  isClosed: %s >  isComment: %s'%(name, str(is_end), str(is_closed), str(is_comment)))
+        # logger.debug(u'%s >  isEnd: %s >  isClosed: %s >  isComment: %s'%(name, unicode(is_end), unicode(is_closed), unicode(is_comment)))
         # logger.debug(u'> %s%s\n'%(rTag[0], rTag[1]))
 
         if name in blockTags:

index.html

@@ -48,13 +48,9 @@
 
           <h3>fanfiction.net / fimfiction.net</h3>
           <p>
-            Fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-          <p>It appears that FimFiction.net is also blocking access from Google
-            App Engine now.
+            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
+            are working again.  I'd ask that users limit the number of
+            stories they download from those sites, thanks.
           </p>
 	  
       {% if authorized %}
@@ -68,8 +64,7 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Alternate URL efiction.trekiverse.org for trekiverse.org/efiction/</li>
-              <li>Minor fixes</li>
+              <li>Fix for possible divide-by-zero in replace_br_with_p.</li>
 	    </ul>
 	  </p>
           
@@ -81,7 +76,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-86.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-90.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -691,13 +691,15 @@ cliches_label:Character Cliches
 ## 'mode'.  'r' to Replace any existing values, 'a' to Add to existing
 ## value (use with tag-like columns), and 'n' for setting on New books
 ## only.  (Default is 'r'.)
+## Literal strings can be set into custom columns using double quotes.
 ## Each metadata=>column mapping must be on a separate line and each
 ## needs to have one space at the start of each line.
 
 #custom_columns_settings:
-# cliches=>#acolumn,r
+# cliches=>#acolumn
 # themes=>#bcolumn,a
 # timeline=>#ccolumn,n
+# "FanFiction"=>#collection
 
 [efiction.esteliel.de]
 ## Site dedicated to these categories/characters/ships
@@ -802,12 +804,24 @@ dislikes_label:Dislikes
 ## Clear FanFiction from defaults, site is original fiction.
 extratags:
 
-extra_valid_entries:size,universe,codes
-#extra_titlepage_entries:size,universe,codes
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice
 
 size_label:Size
 universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
 codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true
 
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The

status.html

@@ -53,13 +53,9 @@
 	  {% if fic.failure %}
           <h3>fanfiction.net / fimfiction.net</h3>
           <p>
-            FYI, fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-          <p>It appears that FimFiction.net is also blocking access from Google
-            App Engine now.
+            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
+            are working again.  I'd ask that users limit the number of
+            stories they download from those sites, thanks.
           </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}