Activate gayauthors.org, support ga authors with space, bump plugin version.

app.yaml

@@ -1,5 +1,5 @@
 # ffd-retief-hrd fanfictiondownloader
-application: fanfictiondownloader
+application: ffd-retief-hrd
 version: 4-3-2
 runtime: python27
 api_version: 1

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 4, 1)
+    version             = (1, 4, 6)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -435,7 +435,9 @@ class OtherTab(QWidget):
                                                   and dynamic[key] is False:
                 dynamic[key] = True
         info_dialog(self, _('Done'),
-                _('Confirmation dialogs have all been reset'), show=True)
+                    _('Confirmation dialogs have all been reset'),
+                    show=True,
+                    show_copy_button=False)
 
 permitted_values = {
     'int' : ['numWords','numChapters'],
@@ -445,6 +447,7 @@ permitted_values = {
     'series' : ['series'],
     'enumeration' : ['category',
                      'genre',
+                     'language',
                      'series',
                      'characters',
                      'status',
@@ -477,6 +480,7 @@ permitted_values['comments'] = permitted_values['enumeration']
 titleLabels = {
     'category':'Category',
     'genre':'Genre',
+    'language':'Language',
     'status':'Status',
     'status-C':'Status:Completed',
     'status-I':'Status:In-Progress',

calibre-plugin/dialogs.py

@@ -60,9 +60,8 @@ class DroppableQTextEdit(QTextEdit):
             return QTextEdit.canInsertFromMimeData(self,source)
 
     def insertFromMimeData(self, source):
-        if source.hasUrls():
-            for u in source.urls():
-                self.append(u.toString())
+        if source.hasText():
+            self.append(source.text())
         else:
             return QTextEdit.insertFromMimeData(self, source)
                             

calibre-plugin/ffdl_plugin.py

@@ -52,9 +52,6 @@ formmapping = {
 
 PLUGIN_ICONS = ['images/icon.png']
 
-sendlists = ["Send to Nook", "Send to Kindle", "Send to Droid", "Add to Nook", "Add to Kindle", "Add to Droid"]
-readlists = ["000"]
-
 class FanFictionDownLoaderPlugin(InterfaceAction):
 
     name = 'FanFictionDownLoader'
@@ -742,11 +739,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
                 old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
             # mi.tags needs to be list, but set kills dups.
-            mi.tags = list(set(list(old_tags)+mi.tags)) 
-        # Set language english, but only if not already set.
-        oldmi = db.get_metadata(book_id,index_is_id=True)
-        if not oldmi.languages:
-            mi.languages=['eng']
+            mi.tags = list(set(list(old_tags)+mi.tags))
+
+        if 'langcode' in book['all_metadata']:
+            mi.languages=[book['all_metadata']['langcode']]
+        else:
+            # Set language english, but only if not already set.
+            oldmi = db.get_metadata(book_id,index_is_id=True)
+            if not oldmi.languages:
+                mi.languages=['eng']
+                
         db.set_metadata(book_id,mi)
 
         # do configured column updates here.
@@ -971,40 +973,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         except:
             return None;
 
-
-
-def get_job_details(job):
-    '''
-    Convert the job result into a set of parameters including a detail message
-    summarising the success of the extraction operation.
-    This is used by both the threaded and worker approaches to extraction
-    '''
-    extracted_ids, same_isbn_ids, failed_ids, no_format_ids = job.result
-    if not hasattr(job, 'html_details'):
-        job.html_details = job.details
-    det_msg = []
-    for i, title in failed_ids:
-        if i in no_format_ids:
-            msg = title + ' (No formats)'
-        else:
-            msg = title + ' (ISBN not found)'
-        det_msg.append(msg)
-    if same_isbn_ids:
-        if det_msg:
-            det_msg.append('----------------------------------')
-        for i, title in same_isbn_ids:
-            msg = title + ' (Same ISBN)'
-            det_msg.append(msg)
-    if len(extracted_ids) > 0:
-        if det_msg:
-            det_msg.append('----------------------------------')
-        for i, title, last_modified, isbn in extracted_ids:
-            msg = '%s (Extracted %s)'%(title, isbn)
-            det_msg.append(msg)
-
-    det_msg = '\n'.join(det_msg)
-    return extracted_ids, same_isbn_ids, failed_ids, det_msg
-
 def get_url_list(urls):
     def f(x):
         if x.strip(): return True

defaults.ini

@@ -36,6 +36,7 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
+language_label:Language
 characters_label:Characters
 series_label:Series
 ## Completed/In-Progress
@@ -67,7 +68,7 @@ version_label:FFDL Version
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -176,7 +177,7 @@ output_css:
 
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true

fanficdownloader/adapters/__init__.py

@@ -44,6 +44,8 @@ import adapter_twiwritenet
 import adapter_whoficcom
 import adapter_siyecouk
 import adapter_archiveofourownorg
+import adapter_ficbooknet
+import adapter_gayauthorsorg
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -134,15 +134,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 		
         a = metasoup.find('dd',{'class':"fandom tags"})
         fandoms = a.findAll('a',{'class':"tag"})
-        fandomstext = [fandom.string for fandom in fandoms]
-        for fandom in fandomstext:
+        for fandom in fandoms:
             self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
         if a != None:
             warnings = a.findAll('a',{'class':"tag"})
-            warningstext = [warning.string for warning in warnings]
-            for warning in warningstext:
+            for warning in warnings:
                 if warning.string == "Author Chose Not To Use Archive Warnings":
                     warning.string = "No Archive Warnings Apply"
                 if warning.string != "No Archive Warnings Apply":
@@ -151,28 +149,24 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         a = metasoup.find('dd',{'class':"freeform tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
-            genrestext = [genre.string for genre in genres]
-            for genre in genrestext:
+            for genre in genres:
                 self.story.addToList('genre',genre.string)
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
-            genrestext = [genre.string for genre in genres]
-            for genre in genrestext:
+            for genre in genres:
                 if genre != "Gen":
                     self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
         if a != None:
             chars = a.findAll('a',{'class':"tag"})
-            charstext = [char.string for char in chars]
-            for char in charstext:
+            for char in chars:
                 self.story.addToList('characters',char.string)
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
             chars = a.findAll('a',{'class':"tag"})
-            charstext = [char.string for char in chars]
-            for char in charstext:
+            for char in chars:
                 self.story.addToList('characters',char.string)
 		
 
@@ -228,19 +222,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if headnotes != None:
             headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
             if headnotes != None:
-                chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))
+                chapter.append("<b>Author's Note:</b>")
                 chapter.append(headnotes)
         
         chapsumm = soup.find('div', {'id' : "summary"})
         if chapsumm != None:
             chapsumm = chapsumm.find('blockquote')
-            chapter.append(bs.BeautifulSoup("<b>Summary for the Chapter:</b>"))
+            chapter.append("<b>Summary for the Chapter:</b>")
             chapter.append(chapsumm)
         chapnotes = soup.find('div', {'id' : "notes"})
         if chapnotes != None:
             chapnotes = chapnotes.find('blockquote')
             if chapnotes != None:
-                chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
+                chapter.append("<b>Notes for the Chapter:</b>")
                 chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
@@ -252,13 +246,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
         if chapfoot != None:
             chapfoot = chapfoot.find('blockquote')
-            chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
+            chapter.append("<b>Notes for the Chapter:</b>")
             chapter.append(chapfoot)
 		
         footnotes = soup.find('div', {'id' : "work_endnotes"})
         if footnotes != None:
             footnotes = footnotes.find('blockquote')
-            chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))
+            chapter.append("<b>Author's Note:</b>")
             chapter.append(footnotes)
 			
         if None == soup:

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -74,6 +74,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
+            #print("\n===================\n%s\n===================\n"%data)
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -83,8 +84,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             
         if "Unable to locate story with id of " in data:
             raise exceptions.StoryDoesNotExist(url)
-            
-        if "Chapter not found. Please check to see you are not using an outdated url." in data:
+
+        # some times "Chapter not found...", sometimes "Chapter text not found..."
+        if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
         try:
@@ -102,7 +104,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                                            chapcount+1)
             print('=Trying newer chapter: %s' % tryurl)
             newdata = self._fetchUrl(tryurl)
-            if "Chapter not found. Please check to see you are not using an outdated url." \
+            if "not found. Please check to see you are not using an outdated url." \
                     not in newdata:
                 print('=======Found newer chapter: %s' % tryurl)
                 soup = bs.BeautifulSoup(newdata)
@@ -140,7 +142,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 continue
             if 'var storyid' in script.string:
                 for line in script.string.split('\n'):
-                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;$",line)
+                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;\r?$",line)
                     if m == None : continue
                     var,value = m.groups()
                     # remove javascript escaping from values.
@@ -201,7 +203,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         metatext = a.findNext(text=re.compile(r' - Reviews:'))
         if metatext == None: # indicates there's no Reviews, look for id: instead.
             metatext = a.findNext(text=re.compile(r' - id:'))
-        #print("========= metatext:\n%s"%metatext)
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
@@ -215,7 +216,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
         # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
         # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?:[ ]+with characters (?P<char1>.*?\.?)(?: & (?P<char2>.*?\.?))?\. )?",
+        # <meta name="description" content="Chapter 1 of a Harry Potter and Transformers  - Humor/Adventure crossover fanfiction  with characters: Harry P. & Ironhide. IT’s one thing to be tossed thru the Veil for something he didn’t do. It was quite another to wake in his animigus form in a world not his own. Harry just knew someone was laughing at him somewhere. Mech/Mech pairings inside..">
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
                      soup.find('meta',{'name':'description'})['content'])
         if m != None:
             genres=m.group('genres')
@@ -225,7 +227,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 for g in genres.split('/'):
                     self.story.addToList('genre',g)
 
-            if m.group('char1') != None:
+            if m.group('chars') != None:
+
                 # At this point we've proven that there's character(s)
                 # We can't reliably parse characters out of meta name="description".
                 # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
@@ -233,12 +236,16 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 # reviewstext can take form of:
                 # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
                 # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
+                # - English - Humor/Adventure -  Harry P. & Ironhide - Reviews:
                 mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
                               metatext)
                 chars = mc.group("chars")
                 for c in chars.split(' & '):
                     self.story.addToList('characters',c)
-        
+        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
+        if m.group('lang') != None:
+            self.story.setMetadata('language',m.group('lang'))
+                    
         return
 
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import datetime
+import logging
+import re
+import urllib2
+from .. import translit
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+
+def getClass():
+    return FicBookNetAdapter
+
+
+class FicBookNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fbn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %m %Y" 
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ficbook.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/readfic/12345"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        url=self.url
+        logging.debug("URL: "+url)
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+				
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+		
+        table = soup.find('td',{'width':'50%'})
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
+        
+        # Find authorid and URL from... author url.
+        a = table.find('a')
+        self.story.setMetadata('authorId',a.text) # Author's name is unique
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.text)
+        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
+
+        # Find the chapters:
+        chapters = soup.find('div', {'class' : 'part_list'})
+        if chapters != None:
+            chapters=chapters.findAll('a', href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+"/\d+#part_content$"))
+            self.story.setMetadata('numChapters',len(chapters))
+            for x in range(0,len(chapters)):
+                chapter=chapters[x]
+                churl='http://'+self.host+chapter['href']
+                self.chapterUrls.append((stripHTML(chapter),churl))
+                if x == 0:
+                    pubdate = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+                if x == len(chapters)-1:
+                    update = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            self.story.setMetadata('numChapters',1)
+            pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
+            update=pubdate
+
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+
+        if not ',' in pubdate:
+            pubdate=datetime.date.today().strftime(self.dateformat)
+        if not ',' in update:
+            update=datetime.date.today().strftime(self.dateformat)
+        pubdate=pubdate.split(',')[0]
+        update=update.split(',')[0]
+		
+        fullmon = {"yanvarya":"01", "января":"01",
+           "fievralya":"02", "февраля":"02",
+           "marta":"03", "марта":"03",
+           "aprielya":"04", "апреля":"04",
+           "maya":"05", "мая":"05",
+           "iyunya":"06", "июня":"06",
+           "iyulya":"07", "июля":"07",
+           "avghusta":"08", "августа":"08",
+           "sentyabrya":"09", "сентября":"09",
+           "oktyabrya":"10", "октября":"10",
+           "noyabrya":"11", "ноября":"11",
+           "diekabrya":"12", "декабря":"12" }
+        for (name,num) in fullmon.items():
+            if name in pubdate:
+                pubdate = pubdate.replace(name,num)
+            if name in update:
+                update = update.replace(name,num)
+
+        self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
+        self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
+        self.story.setMetadata('language','Russian')
+		
+        pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
+        pr='http://'+self.host+pr['href']
+        pr = bs.BeautifulSoup(self._fetchUrl(pr))
+        pr=pr.findAll('div', {'class' : 'part_text'})
+        i=0
+        for part in pr:
+            i=i+len(stripHTML(part).split(' '))
+        self.story.setMetadata('numWords', str(i))
+		
+        i=0
+        fandoms = table.findAll('a', href=re.compile(r'/fanfiction/\w+'))
+        for fandom in fandoms:
+            self.story.addToList('category',fandom.string)
+            i=i+1
+        if i > 1:
+            self.story.addToList('genre', 'Кроссовер')
+		
+        meta=table.findAll('a', href=re.compile(r'/ratings/'))
+        i=0
+        for m in meta:
+            if i == 0:
+                self.story.setMetadata('rating', m.find('b').text)
+                i=1
+            elif i == 1:
+                if not "," in m.nextSibling:
+                    i=2
+                self.story.addToList('genre', m.find('b').text)
+            elif i == 2:
+                self.story.addToList('warnings', m.find('b').text)
+		
+
+        if table.find('span', {'style' : 'color: green'}):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In Progress')
+		
+
+        tags = table.findAll('b')
+        for tag in tags:
+            label = translit.translit(tag.text)
+            if 'Piersonazhi:' in label or 'Персонажи:' in label:
+                chars=tag.nextSibling.string.split(', ')
+                for char in chars:
+                    self.story.addToList('characters',char)
+                break
+				
+        summary=soup.find('span', {'class' : 'urlize'})
+        self.story.setMetadata('description', summary.text)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        chapter = soup.find('div', {'class' : 'public_beta'})
+        if chapter == None:
+            chapter = soup.find('div', {'class' : 'public_beta_disabled'})
+
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return utf8FromSoup(chapter)

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -203,6 +203,10 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 	# our div with poor html inside the story text.
 	data = data.replace('<!-- headerend -->','<crazytagstringnobodywouldstumbleonaccidently id="storytext">').replace('<!-- footerstart -->','</crazytagstringnobodywouldstumbleonaccidently>')
         
+        # problems with some stories confusing Soup.  This is a nasty
+        # hack, but it works.
+        data = data[data.index("<crazytagstringnobodywouldstumbleonaccidently"):]
+
         soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         body = soup.findAll('body') ## some stories use a nested body and body

fanficdownloader/adapters/adapter_gayauthorsorg.py

@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import logging
+import re
+import urllib2
+from urllib import unquote
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+def getClass():
+    return GayAuthorsAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class GayAuthorsAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+       
+		
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        # unqoute, change '_' and ' ' to '-', downcase, and remove non-[a-z0-9-]
+        authid = unquote(self.parsedUrl.path.split('/',)[2])
+        authid = authid.lower().replace('_','-').replace(' ','-')
+        authid = re.sub(r"[^a-z0-9-]","",authid)
+        
+        self.story.setMetadata('authorId',authid)
+        logging.debug("authorId: (%s)"%self.story.getMetadata('authorId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('authorId') + '/' + self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ga')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.gayauthors.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story/author/storytitle"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story/")+r".*?/\w+.*?$"
+
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        msoup = soup.find('div', {'class' : 'story'})
+        if msoup == None:
+            msoup = soup.find('div', {'class' : 'story ispinned'})
+        csoup = soup.find('div', {'id' : 'story_chapters'})
+        
+        ## Title
+        a = msoup.find('span', {'class' : 'title'})
+        title=a.find('span', {'itemprop' : 'name'})
+        self.story.setMetadata('title',title.text)
+        try:
+            # Find Series name from series URL.
+            series = a.find('span',{'class':"description"})
+            series_name = series.find('a')
+            series_name.extract()
+            series_index = int(series.text.split(' ')[1])
+            self.setSeries(series_name.text, series_index)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+        
+        # Find authorid and URL from... author url.
+        a = msoup.find('a', href=re.compile(r'/author/'+self.story.getMetadata('authorId')))
+        self.story.setMetadata('authorUrl',a['href'])
+        self.story.setMetadata('author',a.text)
+		
+
+        # Find the chapters:
+        spans=csoup.findAll('span', {'class' : 'desc chapter-info'})
+        for span in spans:
+            span.extract()
+        for chapter in csoup.findAll('a'):
+            # just in case there's tags, like <i> in chapter titles.
+            a=chapter['href'].split(self.story.getMetadata('author'))
+            a=a[0]+self.story.getMetadata('authorId')+a[1]
+            self.chapterUrls.append((stripHTML(chapter),a))
+
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+		
+        cats = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_0__category\w+$'))
+        for cat in cats:
+            self.story.addToList('category',cat.text)
+		
+        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_1__genre\w+$'))
+        for genre in genres:
+            self.story.addToList('genre',genre.text)
+        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_2__tag\w+$'))
+        for genre in genres:
+            self.story.addToList('genre',genre.text)
+			
+        status = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_3__status\w+$'))
+        self.story.setMetadata('status',status.text)
+		
+        rating = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_4__rating\w+$'))
+        self.story.setMetadata('rating',rating.text)
+		
+        summary = msoup.find('span', {'itemprop' : 'description'})
+        self.story.setMetadata('description',summary.text)
+	
+
+        stats = msoup.find('dl',{'class':'info'})
+        dt = stats.findAll('dt')
+        dd = stats.findAll('dd')
+        for x in range(0,len(dt)):
+            label = dt[x].text
+            value = dd[x].text
+
+            if 'Words:' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Published:' in label:
+                date=stripHTML(value.split(' - ')[0])
+                if ',' in date:
+                    date=datetime.date.today().strftime(self.dateformat)
+                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+				
+            if 'Updated:' in label:
+                date=stripHTML(value.split(' - ')[0])
+                if ',' in date:
+                    date=datetime.date.today().strftime(self.dateformat)
+                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'chapter-content'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return utf8FromSoup(div)

fanficdownloader/adapters/adapter_test1.py

@@ -90,6 +90,16 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
             self.story.setMetadata('status','In-Progress')
         else:
             self.story.setMetadata('status','Completed')
+
+        langs = {
+            0:"English",
+            1:"Russian",
+            2:"French",
+            3:"German",
+            }
+        if idnum < 10:
+            self.story.setMetadata('language',langs[idnum%len(langs)])
+        # greater than 10, no language.
             
         self.setSeries('The Great Test',idnum)
         
@@ -117,7 +127,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
                             ('Chapter 6',self.url+"&chapter=6"),
-                            # ('Chapter 7',self.url+"&chapter=6"),
+                            ('Chapter 7',self.url+"&chapter=6"),
                             # ('Chapter 8',self.url+"&chapter=6"),
                             # ('Chapter 9',self.url+"&chapter=6"),
                             # ('Chapter 0',self.url+"&chapter=6"),

fanficdownloader/htmlcleanup.py

@@ -27,7 +27,7 @@ def _unirepl(match):
     return unichr(value)
 
 def _replaceNumberEntities(data):
-    p = re.compile(r'&#(x?)(\d+);')
+    p = re.compile(r'&#(x?)([0-9a-fA-F]+);')
     return p.sub(_unirepl, data)
 
 def _replaceNotEntities(data):

fanficdownloader/story.py

@@ -19,6 +19,50 @@ import os, re
 
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
+# The list comes from ffnet, the only multi-language site we support
+# at the time of writing.  Values are taken largely from pycountry,
+# but with some corrections and guesses.
+langs = {
+    "English":"en",
+    "Spanish":"es",
+    "French":"fr",
+    "German":"de",
+    "Chinese":"zh",
+    "Japanese":"ja",
+    "Dutch":"nl",
+    "Portuguese":"pt",
+    "Russian":"ru",
+    "Italian":"it",
+    "Bulgarian":"bg",
+    "Polish":"pl",
+    "Hungarian":"hu",
+    "Hebrew":"he",
+    "Arabic":"ar",
+    "Swedish":"sv",
+    "Norwegian":"no",
+    "Danish":"da",
+    "Finnish":"fi",
+    "Filipino":"fil",
+    "Esperanto":"eo",
+    "Hindi":"hi",
+    "Punjabi":"pa",
+    "Farsi":"fa",
+    "Greek":"el",
+    "Romanian":"ro",
+    "Albanian":"sq",
+    "Serbian":"sr",
+    "Turkish":"tr",
+    "Czech":"cs",
+    "Indonesian":"id",
+    "Croatian":"hr",
+    "Catalan":"ca",
+    "Latin":"la",
+    "Korean":"ko",
+    "Vietnamese":"vi",
+    "Thai":"th",
+    "Devanagari":"hi",
+    }
+
 class Story:
     
     def __init__(self):
@@ -33,6 +77,11 @@ class Story:
     def setMetadata(self, key, value):
         ## still keeps < < and &
         self.metadata[key]=conditionalRemoveEntities(value)
+        if key == "language":
+            try:
+                self.metadata['langcode'] = langs[self.metadata[key]]
+            except:
+                self.metadata['langcode'] = 'en'
 
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
@@ -111,7 +160,6 @@ class Story:
     def setReplace(self,replace):
         for line in replace.splitlines():
             if "=>" in line:
-                print("line:%s"%line)
                 self.replacements.append(map( lambda x: x.strip(), line.split("=>") ))
     
 def commaGroups(s):

fanficdownloader/translit.py

@@ -0,0 +1,57 @@
+#-*-coding:utf-8-*-
+# Code taken from http://python.su/forum/viewtopic.php?pid=66946
+import unicodedata
+def is_syllable(letter):
+    syllables = ("A", "E", "I", "O", "U", "a", "e", "i", "o", "u")
+    if letter in syllables:
+        return True
+    return False
+def is_consonant(letter):
+    return not is_syllable(letter)
+def romanize(letter):
+    try:
+        str(letter)
+    except UnicodeEncodeError:
+        pass
+    else:
+        return str(letter)
+    unid = unicodedata.name(letter)
+    exceptions = {"NUMERO SIGN": "No", "LEFT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "DASH": "-"}
+    for name_contains in exceptions:
+        if unid.find(name_contains)!=-1:
+            return exceptions[name_contains]
+    assert(unid.startswith("CYRILLIC"))# Not ready to romanize anything but cyrillics
+    transformation_pairs = {"CYRILLIC CAPITAL LETTER ": str.capitalize, "CYRILLIC SMALL LETTER ": str.lower}
+    func = str.lower
+    for name_contains in transformation_pairs:
+        if unid.find(name_contains)!=-1:
+            func = transformation_pairs[name_contains]
+            unid = unid.replace(name_contains, "")
+    cyrillic_exceptions = {"YERU": "y", "SHORT I": "y", "HARD SIGN": "\'", "SOFT SIGN": "\'", "BYELORUSSIAN-UKRAINIAN I": "i", "GHE WITH UPTURN": "g", "UKRAINIAN IE": "ie", "YU": "yu", "YA": "ya"}
+    for name_contains in cyrillic_exceptions:
+        if unid.find(name_contains)!=-1:
+            return cyrillic_exceptions[name_contains]
+    if all(map(is_syllable, unid)):
+        return func(unid)
+    else:
+        return func(filter(is_consonant, unid))
+def translit(text):
+    output = ""
+    for letter in text:
+        output += romanize(letter)
+    return output
+#def main():
+    #text = u"русск.: Любя, съешь щипцы, — вздохнёт мэр, — кайф жгуч."
+    #print translit(text)
+    #text = u"укр.: Гей, хлопці, не вспію - на ґанку ваша файна їжа знищується бурундучком."
+    #print translit(text)
+    #text = u"болг.: Ах, чудна българска земьо, полюшквай цъфтящи жита."
+    #print translit(text)
+    #text = u"серб.: Неуредне ноћне даме досађивале су Џеку К."
+    #print translit(text)
+    #russk.: Lyubya, s'iesh' shchiptsy, - vzdohniot mer, - kayf zhghuch.
+    #ukr.: Ghiey, hloptsi, nie vspiyu - na ganku vasha fayna yzha znishchuiet'sya burunduchkom.
+    #bolgh.: Ah, chudna b'lgharska ziem'o, polyushkvay ts'ftyashchi zhita.
+    #sierb.: Nieuriednie notshnie damie dosadjivalie su Dzhieku K.
+if __name__=="__main__":
+    main()

fanficdownloader/writers/base_writer.py

@@ -52,6 +52,7 @@ class BaseStoryWriter(Configurable):
         self.validEntries = [
             'category',
             'genre',
+            'language',
             'characters',
             'series',
             'status',
@@ -80,6 +81,7 @@ class BaseStoryWriter(Configurable):
         self.titleLabels = {
             'category':'Category',
             'genre':'Genre',
+            'language':'Language',
             'status':'Status',
             'series':'Series',
             'characters':'Characters',

fanficdownloader/writers/writer_epub.py

@@ -203,7 +203,10 @@ class EpubWriter(BaseStoryWriter):
 
         metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
         metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
-        metadata.appendChild(newTag(contentdom,"dc:language",text="en"))
+        if self.story.getMetadata('langcode') != None:
+            metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
+        else:
+            metadata.appendChild(newTag(contentdom,"dc:language",text='en'))
 
         #  published, created, updated, calibre
         #  Leave calling self.story.getMetadataRaw directly in case date format changes.
@@ -399,4 +402,4 @@ def newTag(dom,name,attrs=None,text=None):
     if( text is not None ):
         tag.appendChild(dom.createTextNode(text))
     return tag
-    
+

index.html

@@ -54,17 +54,9 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>Support for Custom CSS</h3>
+	  <h3>New Site gayauthors.org</h3>
 	  <p>
-	    The CSS included in the HTML and EPUB output formats is now a customizable parameter.
-	  </p>
-	  <h3>Support for Custom Replacement of Metadata</h3>
-	  <p>
-	    There's now a customizable parameter to include a list of regular expressions to replace metadata as you see fit.
-	  </p>
-	  <p>
-	    Examples of how to use both new features can be found in the
-	    <a href="http://www.mobileread.com/forums/showthread.php?p=1962034#post1962034">plugin forum</a>.
+	    Thanks to Ida Leter's hard work, we now support <a href="http://www.gayauthors.org">gayauthors.org</a>, a fanfiction site specializing in gay stories.
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
@@ -239,10 +231,22 @@
 	    <br /><a href="http://archiveofourown.org/works/76366">http://archiveofourown.org/works/76366</a>.
 	    <br /><a href="http://archiveofourown.org/works/76366/chapters/101584">http://archiveofourown.org/works/76366/chapters/101584</a>.
 	  </dd>
-	</dl>	
-
-	
-	A few additional things to know, which will make your life substantially easier:
+	  <dt>ficbook.net(Russian)</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://ficbook.net/readfic/93626">http://ficbook.net/readfic/93626</a>.
+	    <br /><a href="http://ficbook.net/readfic/93626/246417#part_content">http://ficbook.net/readfic/93626/246417#part_content</a>.
+	  </dd>
+	  <dt>gayauthors.org</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://www.gayauthors.org/story/mark-arbour/stvincent">http://www.gayauthors.org/story/mark-arbour/stvincent</a>.
+	    <br /><a href="http://www.gayauthors.org/story/Mark%20Arbour/stvincent/7">http://www.gayauthors.org/story/Mark Arbour/stvincent/7</a>.
+	  </dd>
+	</dl>
+	<p>
+	  A few additional things to know, which will make your life substantially easier:
+	</p>
 	<ol>
 	  <li>
 	    First thing to know: I do not use your Google login and password. In fact, all I know about it is your ID &ndash; password

plugin-defaults.ini

@@ -41,6 +41,7 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
+language_label:Language
 characters_label:Characters
 series_label:Series
 ## Completed/In-Progress
@@ -72,7 +73,7 @@ version_label:FFDL Version
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -154,7 +155,7 @@ output_css:
 
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true