Forgot to add option to web site announcement.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-12
+version: 4-4-13
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 25)
+    version             = (1, 5, 26)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -902,10 +902,18 @@ make_firstimage_cover:true
             if len(lists) < 1 :
                 message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
                 confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
+
+            # Quick demo of how an 'allow send' list might work.
+            # Issues: allow list per send list?  Naming convention?  "send(allow)"
+            # allow_list = rl_plugin.get_book_list("Allow Send to Device")
+            # # intersection of book_ids & allow_list
+            # add_book_ids = list(set(book_ids) & set(allow_list))
+
             for l in lists:
                 if l in rl_plugin.get_list_names():
                     #print("good send l:(%s)"%l)
                     rl_plugin.add_books_to_list(l,
+                                                #add_book_ids,
                                                 book_ids,
                                                 display_warnings=False)
                 else:

defaults.ini

@@ -183,6 +183,9 @@ output_css:
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78
+
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
 
@@ -444,6 +447,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[www.dokuga.com]
+
 [www.fanfiction.net]
 
 [www.ficbook.net]

fanficdownloader/adapters/__init__.py

@@ -65,6 +65,7 @@ import adapter_phoenixsongnet
 import adapter_walkingtheplankorg
 import adapter_ashwindersycophanthexcom
 import adapter_thehexfilesnet
+import adapter_dokugacom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_dokugacom.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DokugaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DokugaComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dkg')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","InuYasha")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'fanfiction' in self.story.getMetadata('section'):
+            self.dateformat = "%d %b %Y"
+        else:
+            self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dokuga.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'align' : 'center'}).find('h3')
+        
+        # Find authorid and URL from... author url.
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        alink='http://'+self.host+aut['href']
+        self.story.setMetadata('authorUrl','http://'+self.host+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        a = a.string[:(len(a.string)-4)]
+        self.story.setMetadata('title',a)
+
+        # Find the chapters:
+        chapters = soup.find('select').findAll('option')
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/1'))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(alink))
+        
+        if 'fanfiction' in self.story.getMetadata('section'):
+            asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
+        
+            #grab the rest of the metadata from the author's page
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+            div=div.nextSibling
+            self.setDescription(url,div)
+        
+            div=div.nextSibling
+            self.story.setMetadata('rating', div.text.split('Rating: ')[1].split('&')[0])
+        
+            iscomp=div.text.split('Status: ')[1].split('&')[0]
+            if 'Complete' in iscomp:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+            
+            self.story.addToList('category', div.text.split('Category: ')[1].split('&')[0])
+            self.story.addToList('category', 'Fanfiction')
+        
+            self.story.setMetadata('datePublished', makeDate(stripHTML(div.text.split('Created: ')[1].split('&')[0]), self.dateformat))
+        
+            self.story.setMetadata('dateUpdated', makeDate(stripHTML(div.text.split('Updated: ')[1]), self.dateformat))
+            
+            div=div.nextSibling.nextSibling
+            self.story.setMetadata('numWords', div.text.split('Words: ')[1].split('&')[0])
+        
+            for genre in div.text.split('Genre: ')[1].split('&')[0].split(', '):
+                self.story.addToList('genre',genre)
+        
+        else:
+            asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+                    
+            div=div.nextSibling.nextSibling
+            self.setDescription(url,div)
+            self.story.addToList('category', 'Spark')
+            
+            div=div.nextSibling.nextSibling
+            self.story.setMetadata('rating', div.text.split('Rating: ')[1].split(' - ')[0])
+            
+            iscomp=div.text.split('Status: ')[1].split(' - ')[0]
+            if 'Complete' in iscomp:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+            
+            for genre in div.text.split('Genre: ')[1].split(' - ')[0].split('/'):
+                self.story.addToList('genre',genre)
+                
+            div=div.nextSibling.nextSibling
+            
+            date=div.text.split('Updated: ')[1].split('      -')[0]
+            self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+        
+            # does not have published date anywhere
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            
+            self.story.setMetadata('numWords', div.text.split('Words ')[1])
+            
+        
+
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'chtext'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -268,8 +268,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         sharediv = soup.find('div', {'class' : 'a2a_kit a2a_default_style'})
         if sharediv:
             sharediv.extract()
-        else:
-            logging.debug('share button div not found')
         
         div = soup.find('div', {'id' : 'storytext'})
         

fanficdownloader/html2text.py

@@ -103,9 +103,10 @@ def onlywhite(line):
             return c is ' '
     return line
 
-def optwrap(text):
+def optwrap(text,wrap_width=BODY_WIDTH):
     """Wrap all paragraphs in the provided text."""
-    if not BODY_WIDTH:
+    
+    if not wrap_width:
         return text
     
     assert wrap, "Requires Python 2.3."
@@ -114,7 +115,7 @@ def optwrap(text):
     for para in text.split("\n"):
         if len(para) > 0:
             if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*':
-                for line in wrap(para, BODY_WIDTH):
+                for line in wrap(para, wrap_width):
                     result += line + "\n"
                 result += "\n"
                 newlines = 2
@@ -423,8 +424,8 @@ def html2text_file(html, out=wrapwrite, baseurl=''):
     h.feed("")
     return h.close()
 
-def html2text(html, baseurl=''):
-    return optwrap(html2text_file(html, None, baseurl))
+def html2text(html, baseurl='', wrap_width=BODY_WIDTH):
+    return optwrap(html2text_file(html, None, baseurl),wrap_width)
 
 if __name__ == "__main__":
     baseurl = ''

fanficdownloader/writers/writer_txt.py

@@ -21,7 +21,7 @@ from textwrap import wrap
 
 from base_writer import *
 
-from ..html2text import html2text, BODY_WIDTH
+from ..html2text import html2text
 
 ## In BaseStoryWriter, we define _write to encode <unicode> objects
 ## back into <string> for true output.  But txt needs to write the
@@ -106,6 +106,12 @@ End file.
 
     def writeStoryImpl(self, out):
 
+        self.wrap_width = self.getConfig('wrap_width')
+        if self.wrap_width == '' or self.wrap_width == '0':
+            self.wrap_width = None
+        else:
+            self.wrap_width = int(self.wrap_width)
+        
         wrapout = KludgeStringIO()
         
         wrapout.write(self.TEXT_FILE_START.substitute(self.story.metadata))
@@ -131,15 +137,19 @@ End file.
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(self.TEXT_CHAPTER_START.substitute({'chapter':title, 'index':index+1})))))
-                self._write(out,self.lineends(html2text(html)))
+                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
 
         self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.metadata))))
 
     def wraplines(self, text):
+        
+        if not self.wrap_width:
+            return text
+        
         result=''
         for para in text.split("\n"):
             first=True
-            for line in wrap(para, BODY_WIDTH):
+            for line in wrap(para, self.wrap_width):
                 if first:
                     first=False
                 else:

index.html

@@ -54,9 +54,13 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>Fix for fanfiction.net Genre, Characters and Status</h3>
+	  <h3>New Site</h3>
 	  <p>
-	    fanfiction.net changed their format yesterday or today(May 26/27) enough to break capture of Genre, Characters and Status.  This version fixes it.
+	    New site www.dokuga.com added.  Thanks, Ida.
+	  </p>
+	  <h3>New Option</h3>
+	  <p>
+	    New configuration option under [txt], wrap_width.  Use to control or disable word wrap in text mode.
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -66,7 +70,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-11.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-12.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -358,6 +362,12 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://thehexfiles.net/viewstory.php?sid=1234">http://thehexfiles.net/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>www.dokuga.com</dt>
+	  <dd>
+	    Use the URL of any story chapter, such as
+	    <br /><a href="http://www.dokuga.com/fanfiction/story/1234/1">http://www.dokuga.com/fanfiction/story/1234/1</a> or
+	    <br /><a href="http://www.dokuga.com/spark/story/1234/1">http://www.dokuga.com/spark/story/1234/1</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

plugin-defaults.ini

@@ -172,6 +172,9 @@ output_css:
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78
+
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
 
@@ -430,6 +433,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[www.dokuga.com]
+
 [www.fanfiction.net]
 
 [www.ficbook.net]