Forgot to add option to web site announcement.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-11
+version: 4-4-13
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 23)
+    version             = (1, 5, 26)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -48,6 +48,7 @@ all_prefs.defaults['deleteotherforms'] = False
 all_prefs.defaults['adddialogstaysontop'] = False
 all_prefs.defaults['includeimages'] = False
 all_prefs.defaults['lookforurlinhtml'] = False
+all_prefs.defaults['injectseries'] = False
 
 all_prefs.defaults['send_lists'] = ''
 all_prefs.defaults['read_lists'] = ''
@@ -75,6 +76,7 @@ copylist = ['personal.ini',
             'adddialogstaysontop',
             'includeimages',
             'lookforurlinhtml',
+            'injectseries',
             'gcnewonly',
             'gc_site_settings',
             'allow_gc_from_ini']
@@ -181,6 +183,7 @@ class ConfigWidget(QWidget):
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
+        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -320,6 +323,11 @@ class BasicTab(QWidget):
         self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
         self.l.addWidget(self.lookforurlinhtml)
 
+        self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
+        self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
+        self.injectseries.setChecked(prefs['injectseries'])
+        self.l.addWidget(self.injectseries)
+
         self.l.insertStretch(-1)
         
     def set_collisions(self):

calibre-plugin/ffdl_plugin.py

@@ -572,7 +572,13 @@ make_firstimage_cover:true
             # this behavior matches how epubs come out when imported
             # dateCreated == packaged--epub/etc created.
             book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
-        
+
+        if book_id != None and prefs['injectseries']:
+            mi = db.get_metadata(book_id,index_is_id=True)
+            if not book['series'] and mi.series != None:
+                book['calibre_series'] = (mi.series,mi.series_index)
+                print("calibre_series:%s [%s]"%book['calibre_series'])
+            
         if book['good']: # there shouldn't be any !'good' books at this point.
             # if still 'good', make a temp file to write the output to.
             tmp = PersistentTemporaryFile(prefix='new-%s-'%book['calibre_id'],
@@ -795,7 +801,7 @@ make_firstimage_cover:true
             coldef = custom_columns[col]
             if not meta.startswith('status-') and meta not in book['all_metadata'] or \
                     meta.startswith('status-') and 'status' not in book['all_metadata']:
-                print("No value for %s, skipping."%meta)
+                print("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
                 continue
             if meta not in permitted_values[coldef['datatype']]:
                 print("%s not a valid column type for %s, skipping."%(col,meta))
@@ -815,7 +821,6 @@ make_firstimage_cover:true
 
         db.commit()
         
-        print("book['added']:%s"%book['added'])
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
             gc_plugin = self.gui.iactions['Generate Cover']
             setting_name = None
@@ -897,10 +902,18 @@ make_firstimage_cover:true
             if len(lists) < 1 :
                 message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
                 confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
+
+            # Quick demo of how an 'allow send' list might work.
+            # Issues: allow list per send list?  Naming convention?  "send(allow)"
+            # allow_list = rl_plugin.get_book_list("Allow Send to Device")
+            # # intersection of book_ids & allow_list
+            # add_book_ids = list(set(book_ids) & set(allow_list))
+
             for l in lists:
                 if l in rl_plugin.get_list_names():
                     #print("good send l:(%s)"%l)
                     rl_plugin.add_books_to_list(l,
+                                                #add_book_ids,
                                                 book_ids,
                                                 display_warnings=False)
                 else:

calibre-plugin/jobs.py

@@ -113,6 +113,11 @@ def do_download_for_worker(book,options):
         adapter.password = book['password']
         
         story = adapter.getStoryMetadataOnly()
+        if 'calibre_series' in book:
+            print("calibre_series:%s [%d]"%book['calibre_series'])
+            adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
+        else:
+            print("no calibre_series")
         writer = writers.getWriter(options['fileform'],adapter.config,adapter)
 
         outfile = book['outfile']

defaults.ini

@@ -183,6 +183,9 @@ output_css:
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78
+
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
 
@@ -409,6 +412,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[thehexfiles.net]
+
 [thequidditchpitch.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -442,6 +447,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[www.dokuga.com]
+
 [www.fanfiction.net]
 
 [www.ficbook.net]

fanficdownloader/adapters/__init__.py

@@ -64,6 +64,8 @@ import adapter_occlumencysycophanthexcom
 import adapter_phoenixsongnet
 import adapter_walkingtheplankorg
 import adapter_ashwindersycophanthexcom
+import adapter_thehexfilesnet
+import adapter_dokugacom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_dokugacom.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DokugaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DokugaComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dkg')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","InuYasha")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'fanfiction' in self.story.getMetadata('section'):
+            self.dateformat = "%d %b %Y"
+        else:
+            self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dokuga.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'align' : 'center'}).find('h3')
+        
+        # Find authorid and URL from... author url.
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        alink='http://'+self.host+aut['href']
+        self.story.setMetadata('authorUrl','http://'+self.host+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        a = a.string[:(len(a.string)-4)]
+        self.story.setMetadata('title',a)
+
+        # Find the chapters:
+        chapters = soup.find('select').findAll('option')
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/1'))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(alink))
+        
+        if 'fanfiction' in self.story.getMetadata('section'):
+            asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
+        
+            #grab the rest of the metadata from the author's page
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+            div=div.nextSibling
+            self.setDescription(url,div)
+        
+            div=div.nextSibling
+            self.story.setMetadata('rating', div.text.split('Rating: ')[1].split('&')[0])
+        
+            iscomp=div.text.split('Status: ')[1].split('&')[0]
+            if 'Complete' in iscomp:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+            
+            self.story.addToList('category', div.text.split('Category: ')[1].split('&')[0])
+            self.story.addToList('category', 'Fanfiction')
+        
+            self.story.setMetadata('datePublished', makeDate(stripHTML(div.text.split('Created: ')[1].split('&')[0]), self.dateformat))
+        
+            self.story.setMetadata('dateUpdated', makeDate(stripHTML(div.text.split('Updated: ')[1]), self.dateformat))
+            
+            div=div.nextSibling.nextSibling
+            self.story.setMetadata('numWords', div.text.split('Words: ')[1].split('&')[0])
+        
+            for genre in div.text.split('Genre: ')[1].split('&')[0].split(', '):
+                self.story.addToList('genre',genre)
+        
+        else:
+            asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+                    
+            div=div.nextSibling.nextSibling
+            self.setDescription(url,div)
+            self.story.addToList('category', 'Spark')
+            
+            div=div.nextSibling.nextSibling
+            self.story.setMetadata('rating', div.text.split('Rating: ')[1].split(' - ')[0])
+            
+            iscomp=div.text.split('Status: ')[1].split(' - ')[0]
+            if 'Complete' in iscomp:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+            
+            for genre in div.text.split('Genre: ')[1].split(' - ')[0].split('/'):
+                self.story.addToList('genre',genre)
+                
+            div=div.nextSibling.nextSibling
+            
+            date=div.text.split('Updated: ')[1].split('      -')[0]
+            self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+        
+            # does not have published date anywhere
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            
+            self.story.setMetadata('numWords', div.text.split('Words ')[1])
+            
+        
+
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'chtext'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -205,21 +205,29 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if metatext == None: # indicates there's no Reviews, look for id: instead.
             metatext = a.findNext(text=re.compile(r' - id:'))
 
+        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
+        if m.group('lang') != None:
+            self.story.setMetadata('language',m.group('lang'))
+                    
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
+        if 'Complete' in soup.find(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
 
+
         # Parse genre(s) from <meta name="description" content="..."
         # <meta name="description" content="A Transformers/Beast Wars  - Humor fanfiction with characters Prowl & Sideswipe. Story summary: Sideswipe is bored. Prowl appears to be so, too  or at least, Sideswipe thinks he looks bored . So Sideswipe entertains them. After all, what's more fun than a race? Song-fic.">
         # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
         # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
         # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
         # <meta name="description" content="Chapter 1 of a Harry Potter and Transformers  - Humor/Adventure crossover fanfiction  with characters: Harry P. & Ironhide. IT’s one thing to be tossed thru the Veil for something he didn’t do. It was quite another to wake in his animigus form in a world not his own. Harry just knew someone was laughing at him somewhere. Mech/Mech pairings inside..">
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
+        # <meta name="description" content="Chapter 1 of a SpongeBob SquarePants - Romance/Humor fanfiction with characters SpongeBob. Bob Esponja tiene un admirador secreto ¿quien será?.">
+        # Chapter 1 of a SpongeBob SquarePants - Romance/Humor fanfiction with characters SpongeBob. Bob Esponja tiene un admirador secreto ¿quien será?.  update existing id:1684
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  ?(?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
                      soup.find('meta',{'name':'description'})['content'])
+        #print("meta desc:%s"%soup.find('meta',{'name':'description'})['content'])
         if m != None:
             genres=m.group('genres')
             if genres != None:
@@ -238,15 +246,13 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
                 # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
                 # - English - Humor/Adventure -  Harry P. & Ironhide - Reviews:
-                mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
+                # - Spanish - Romance/Humor - SpongeBob - Reviews:
+                #print("metatext:%s"%metatext)
+                mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? ?(?P<chars>.+?) - (Reviews|Updated|Published)",
                               metatext)
                 chars = mc.group("chars")
-                for c in chars.split(' & '):
-                    self.story.addToList('characters',c)
-        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
-        if m.group('lang') != None:
-            self.story.setMetadata('language',m.group('lang'))
-                    
+                for c in chars.split('&'):
+                    self.story.addToList('characters',c.strip())
         return
 
     def getChapterText(self, url):
@@ -262,8 +268,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         sharediv = soup.find('div', {'class' : 'a2a_kit a2a_default_style'})
         if sharediv:
             sharediv.extract()
-        else:
-            logging.debug('share button div not found')
         
         div = soup.find('div', {'id' : 'storytext'})
         

fanficdownloader/adapters/adapter_test1.py

@@ -90,17 +90,16 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         else:
             self.story.setMetadata('status','Completed')
 
-        langs = {
-            0:"English",
-            1:"Russian",
-            2:"French",
-            3:"German",
-            }
+        # greater than 10, no language or series.
         if idnum < 10:
+            langs = {
+                0:"English",
+                1:"Russian",
+                2:"French",
+                3:"German",
+                }
             self.story.setMetadata('language',langs[idnum%len(langs)])
-        # greater than 10, no language.
-            
-        self.setSeries('The Great Test',idnum)
+            self.setSeries('The Great Test',idnum)
         
         self.story.setMetadata('rating','Tweenie')
         
@@ -161,7 +160,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
             text=u'''
 <div>
 <h3>Prologue</h3>
-<p>This is a fake adapter for testing purposes.  Different storyId's will give different errors:</p>
+<p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
@@ -171,18 +170,17 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
-<p>And other storyId will succeed with the same output.</p>
+<p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
 '''
         else:
             text=u'''
 <div>
 <h3>Chapter title from site</h3>
-<p><center>Centered text</center></p>
 <p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 br breaks<br><br>
 Puella Magi Madoka Magica/魔法少女まどか★マギカ
-<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim &amp; Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a><br/>
+<!-- a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim &amp; Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a --><br/>
 br breaks<br><br>
 Don't&#8212e;ver&#8212d;o&#8212;that&#8212a;gain, &#27861; &#xE9;
 <hr>

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheHexFilesNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheHexFilesNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','thf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thehexfiles.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'cellspacing' : '4'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+			
+        words = info.find(text=re.compile('Words:')).split('|')[1].split(': ')[1]
+        self.story.setMetadata('numWords', words)
+		
+        comp = info.find('span', {'class' : 'completed'}).string.split(': ')[1]
+        if 'Yes' in comp:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+						
+        summary = info.find('td', {'class' : 'summary'})
+        self.setDescription(url,summary)
+		
+        rating=stripHTML(info.find('td', {'align' : 'left'})).split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        labels = info.findAll('td', {'width' : '10%'})
+        values = info.findAll('td', {'width' : '40%'})
+        for i in range(0,len(labels)):
+            value = stripHTML(values[i])
+            label = stripHTML(labels[i])
+
+            if 'Genres' in label:
+                genres = value.split(', ')
+                for genre in genres:
+                    if genre != 'none':
+                        self.story.addToList('genre',genre)
+						
+
+            if 'Warnings' in label:
+                warnings = value.split(', ')
+                for warning in warnings:
+                    if warning != 'none':
+                        self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        for a in soup.findAll('table'):
+            a.extract()
+
+        if None == soup:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/base_adapter.py

@@ -265,7 +265,7 @@ class BaseSiteAdapter(Configurable):
     # Just for series, in case we choose to change how it's stored or represented later.
     def setSeries(self,name,num):
         if self.getConfig('collect_series'):
-            self.story.setMetadata('series','%s [%s]'%(name, num))
+            self.story.setMetadata('series','%s [%s]'%(name, int(num)))
 
     def setDescription(self,url,svalue):
         #print("\n\nsvalue:\n%s\n"%svalue)

fanficdownloader/html2text.py

@@ -103,9 +103,10 @@ def onlywhite(line):
             return c is ' '
     return line
 
-def optwrap(text):
+def optwrap(text,wrap_width=BODY_WIDTH):
     """Wrap all paragraphs in the provided text."""
-    if not BODY_WIDTH:
+    
+    if not wrap_width:
         return text
     
     assert wrap, "Requires Python 2.3."
@@ -114,7 +115,7 @@ def optwrap(text):
     for para in text.split("\n"):
         if len(para) > 0:
             if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*':
-                for line in wrap(para, BODY_WIDTH):
+                for line in wrap(para, wrap_width):
                     result += line + "\n"
                 result += "\n"
                 newlines = 2
@@ -423,8 +424,8 @@ def html2text_file(html, out=wrapwrite, baseurl=''):
     h.feed("")
     return h.close()
 
-def html2text(html, baseurl=''):
-    return optwrap(html2text_file(html, None, baseurl))
+def html2text(html, baseurl='', wrap_width=BODY_WIDTH):
+    return optwrap(html2text_file(html, None, baseurl),wrap_width)
 
 if __name__ == "__main__":
     baseurl = ''

fanficdownloader/writers/writer_txt.py

@@ -21,7 +21,7 @@ from textwrap import wrap
 
 from base_writer import *
 
-from ..html2text import html2text, BODY_WIDTH
+from ..html2text import html2text
 
 ## In BaseStoryWriter, we define _write to encode <unicode> objects
 ## back into <string> for true output.  But txt needs to write the
@@ -106,6 +106,12 @@ End file.
 
     def writeStoryImpl(self, out):
 
+        self.wrap_width = self.getConfig('wrap_width')
+        if self.wrap_width == '' or self.wrap_width == '0':
+            self.wrap_width = None
+        else:
+            self.wrap_width = int(self.wrap_width)
+        
         wrapout = KludgeStringIO()
         
         wrapout.write(self.TEXT_FILE_START.substitute(self.story.metadata))
@@ -131,15 +137,19 @@ End file.
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(self.TEXT_CHAPTER_START.substitute({'chapter':title, 'index':index+1})))))
-                self._write(out,self.lineends(html2text(html)))
+                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
 
         self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.metadata))))
 
     def wraplines(self, text):
+        
+        if not self.wrap_width:
+            return text
+        
         result=''
         for para in text.split("\n"):
             first=True
-            for line in wrap(para, BODY_WIDTH):
+            for line in wrap(para, self.wrap_width):
                 if first:
                     first=False
                 else:

index.html

@@ -54,19 +54,13 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>More New Sites</h3>
+	  <h3>New Site</h3>
 	  <p>
-	    Ida's been busy!  We now have support for an additional 8 new sites:
-	    <ul>
-	      <li>ashwinder.sycophanthex.com</li>
-	      <li>chaos.sycophanthex.com</li>
-	      <li>erosnsappho.sycophanthex.com</li>
-	      <li>lumos.sycophanthex.com</li>
-	      <li>occlumency.sycophanthex.com</li>
-	      <li>dramione.org</li>
-	      <li>www.phoenixsong.net</li>
-	      <li>www.walkingtheplank.org</li>
-	    </ul>
+	    New site www.dokuga.com added.  Thanks, Ida.
+	  </p>
+	  <h3>New Option</h3>
+	  <p>
+	    New configuration option under [txt], wrap_width.  Use to control or disable word wrap in text mode.
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -76,7 +70,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-10.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-12.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -363,6 +357,17 @@
 	    Use the URL of the story's first chapter, such as
 	    <br /><a href="http://www.walkingtheplank.org/archive/viewstory.php?sid=1234">http://www.walkingtheplank.org/archive/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>thehexfiles.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://thehexfiles.net/viewstory.php?sid=1234">http://thehexfiles.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.dokuga.com</dt>
+	  <dd>
+	    Use the URL of any story chapter, such as
+	    <br /><a href="http://www.dokuga.com/fanfiction/story/1234/1">http://www.dokuga.com/fanfiction/story/1234/1</a> or
+	    <br /><a href="http://www.dokuga.com/spark/story/1234/1">http://www.dokuga.com/spark/story/1234/1</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

plugin-defaults.ini

@@ -172,6 +172,9 @@ output_css:
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78
+
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
 
@@ -395,6 +398,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[thehexfiles.net]
+
 [thequidditchpitch.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -428,6 +433,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[www.dokuga.com]
+
 [www.fanfiction.net]
 
 [www.ficbook.net]