Add skip header kludge to fictionalleyorg, bump plugin to 1.4.4.

Many(most?) of the sites ignore gzip because of our User-Agent. ffnet honors it.
Add progress bar while getting URLs or getting URLs for update.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-3-1
+version: 4-3-2
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 3, 2)
+    version             = (1, 4, 4)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import traceback, copy
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont,
                       QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant)
 
 from calibre.gui2 import dynamic, info_dialog
@@ -58,10 +58,7 @@ copylist = ['personal.ini',
             'updatedefault',
             'fileform',
             'collision',
-            'deleteotherforms',
-            'addtolists',
-            'addtoreadlists',
-            'addtolistsonread']
+            'deleteotherforms']
 
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
@@ -293,6 +290,11 @@ class PersonalIniTab(QWidget):
         self.l.addWidget(self.label)
 
         self.ini = QTextEdit(self)
+        try:
+            self.ini.setFont(QFont("Courier",
+                                   self.plugin_action.gui.font().pointSize()+1));
+        except Exception as e:
+            print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
@@ -324,6 +326,11 @@ class ShowDefaultsIniDialog(QDialog):
         
         self.ini = QTextEdit(self)
         self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        try:
+            self.ini.setFont(QFont("Courier",
+                                   get_gui().font().pointSize()+1));
+        except Exception as e:
+            print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(text)
         self.ini.setReadOnly(True)
@@ -428,7 +435,9 @@ class OtherTab(QWidget):
                                                   and dynamic[key] is False:
                 dynamic[key] = True
         info_dialog(self, _('Done'),
-                _('Confirmation dialogs have all been reset'), show=True)
+                    _('Confirmation dialogs have all been reset'),
+                    show=True,
+                    show_copy_button=False)
 
 permitted_values = {
     'int' : ['numWords','numChapters'],
@@ -438,6 +447,7 @@ permitted_values = {
     'series' : ['series'],
     'enumeration' : ['category',
                      'genre',
+                     'language',
                      'series',
                      'characters',
                      'status',
@@ -470,6 +480,7 @@ permitted_values['comments'] = permitted_values['enumeration']
 titleLabels = {
     'category':'Category',
     'genre':'Genre',
+    'language':'Language',
     'status':'Status',
     'status-C':'Status:Completed',
     'status-I':'Status:In-Progress',

calibre-plugin/ffdl_plugin.py

@@ -52,9 +52,6 @@ formmapping = {
 
 PLUGIN_ICONS = ['images/icon.png']
 
-sendlists = ["Send to Nook", "Send to Kindle", "Send to Droid", "Add to Nook", "Add to Kindle", "Add to Droid"]
-readlists = ["000"]
-
 class FanFictionDownLoaderPlugin(InterfaceAction):
 
     name = 'FanFictionDownLoader'
@@ -119,6 +116,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def about_to_show_menu(self):
         self.rebuild_menus()
 
+    def library_changed(self, db):
+        # We need to reset our menus after switching libraries
+        self.rebuild_menus()
+        
     def rebuild_menus(self):
         with self.menus_lock:
             # Show the config dialog
@@ -239,22 +240,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
     def get_list_urls(self):
         if len(self.gui.library_view.get_selected_ids()) > 0:
-            url_list = []
-            for book_id in self.gui.library_view.get_selected_ids():
-                url = self._get_story_url(self.gui.current_db, book_id)
-                if url != None:
-                    url_list.append(url)
+            book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
 
-            if url_list:
-                d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
-                d.setWindowIcon(get_icon('bookmarks.png'))
-                d.exec_()
-            else:
-                info_dialog(self.gui, _('List of URLs'),
-                            _('No URLs found in selected books.'),
-                            show=True,
-                            show_copy_button=False)
+            LoopProgressDialog(self.gui,
+                               book_list,
+                               partial(self._get_story_url_for_list, db=self.gui.current_db),
+                               self._finish_get_list_urls,
+                               init_label="Collecting URLs for stories...",
+                               win_title="Get URLs for stories",
+                               status_prefix="URL retrieved")
             
+    def _get_story_url_for_list(self,book,db=None):
+        book['url'] = self._get_story_url(db,book['calibre_id'])
+        if book['url'] == None:
+            book['good']=False
+        else:
+            book['good']=True
+            
+    def _finish_get_list_urls(self, book_list):
+        url_list = [ x['url'] for x in book_list if x['good'] ]
+        if url_list:
+            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
+            d.setWindowIcon(get_icon('bookmarks.png'))
+            d.exec_()
+        else:
+            info_dialog(self.gui, _('List of URLs'),
+                        _('No URLs found in selected books.'),
+                        show=True,
+                        show_copy_button=False)
+                
     def add_dialog(self):
 
         #print("add_dialog()")
@@ -290,17 +304,29 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if len(self.gui.library_view.get_selected_ids()) == 0:
             return
         #print("update_existing()")
-        previous = self.gui.library_view.currentIndex()
+        
         db = self.gui.current_db
-        book_ids = self.gui.library_view.get_selected_ids()
-        books = self._convert_calibre_ids_to_books(db, book_ids)
+        book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
+        #book_ids = self.gui.library_view.get_selected_ids()
+
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._populate_book_from_calibre_id, db=self.gui.current_db),
+                           self._update_existing_2,
+                           init_label="Collecting stories for update...",
+                           win_title="Get stories for updates",
+                           status_prefix="URL retrieved")            
+        
+        #books = self._convert_calibre_ids_to_books(db, book_ids)
         #print("update books:%s"%books)
 
+    def _update_existing_2(self,book_list):
+        
         d = UpdateExistingDialog(self.gui,
                                  'Update Existing List',
                                  prefs,
                                  self.qaction.icon(),
-                                 books,
+                                 book_list,
                                  )
         d.exec_()
         if d.result() != d.Accepted:
@@ -339,10 +365,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
         
-        LoopProgressDialog(self.gui,
-                           books,
-                           partial(self.get_metadata_for_book, options = options),
-                           partial(self.start_download_list, options = options))
+        if 0 < len(filter(lambda x : x['good'], books)):
+            LoopProgressDialog(self.gui,
+                               books,
+                               partial(self.get_metadata_for_book, options = options),
+                               partial(self.start_download_list, options = options))
         # LoopProgressDialog calls get_metadata_for_book for each 'good' story,
         # get_metadata_for_book updates book for each,
         # LoopProgressDialog calls start_download_list at the end which goes
@@ -658,14 +685,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             total_good = len(good_list)
 
             self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
-            
-            LoopProgressDialog(self.gui,
-                               good_list,
-                               partial(self._update_book, options=options, db=self.gui.current_db),
-                               partial(self._update_books_completed, options=options),
-                               init_label="Updating calibre for stories...",
-                               win_title="Update calibre for stories",
-                               status_prefix="Updated")
+
+            if total_good > 0:
+                LoopProgressDialog(self.gui,
+                                   good_list,
+                                   partial(self._update_book, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options),
+                                   init_label="Updating calibre for stories...",
+                                   win_title="Update calibre for stories",
+                                   status_prefix="Updated")
 
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -711,11 +739,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
                 old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
             # mi.tags needs to be list, but set kills dups.
-            mi.tags = list(set(list(old_tags)+mi.tags)) 
-        # Set language english, but only if not already set.
-        oldmi = db.get_metadata(book_id,index_is_id=True)
-        if not oldmi.languages:
-            mi.languages=['eng']
+            mi.tags = list(set(list(old_tags)+mi.tags))
+
+        if 'langcode' in book['all_metadata']:
+            mi.languages=[book['all_metadata']['langcode']]
+        else:
+            # Set language english, but only if not already set.
+            oldmi = db.get_metadata(book_id,index_is_id=True)
+            if not oldmi.languages:
+                mi.languages=['eng']
+                
         db.set_metadata(book_id,mi)
 
         # do configured column updates here.
@@ -854,16 +887,30 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         self._set_book_url_and_comment(book,url)
         return book
         
-    
-    def _convert_calibre_ids_to_books(self, db, ids):
-        books = []
-        for book_id in ids:
-            books.append(self._convert_calibre_id_to_book(db,book_id))
-        return books
-            
-    def _convert_calibre_id_to_book(self, db, book_id):
-        mi = db.get_metadata(book_id, index_is_id=True)
+    def _convert_id_to_book(self, idval, good=True):
         book = {}
+        book['good'] = good
+        book['calibre_id'] = idval
+        book['title'] = 'Unknown'
+        book['author'] = 'Unknown'
+        book['author_sort'] = 'Unknown'
+            
+        book['comment'] = ''
+        book['url'] = ''
+        book['added'] = False
+        
+        return book
+        
+    
+    # def _convert_calibre_ids_to_books(self, db, ids):
+    #     books = []
+    #     for book_id in ids:
+    #         books.append(self._convert_calibre_id_to_book(db,book_id))
+    #     return books
+            
+    def _populate_book_from_calibre_id(self, book, db=None):
+        mi = db.get_metadata(book['calibre_id'], index_is_id=True)
+        #book = {}
         book['good'] = True
         book['calibre_id'] = mi.id
         book['title'] = mi.title
@@ -873,10 +920,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['url'] = ""
         book['added'] = False
         
-        url = self._get_story_url(db,book_id)
+        url = self._get_story_url(db,book['calibre_id'])
         self._set_book_url_and_comment(book,url)
-
-        return book
+        #return book
 
     def _set_book_url_and_comment(self,book,url):
         if not url:
@@ -927,40 +973,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         except:
             return None;
 
-
-
-def get_job_details(job):
-    '''
-    Convert the job result into a set of parameters including a detail message
-    summarising the success of the extraction operation.
-    This is used by both the threaded and worker approaches to extraction
-    '''
-    extracted_ids, same_isbn_ids, failed_ids, no_format_ids = job.result
-    if not hasattr(job, 'html_details'):
-        job.html_details = job.details
-    det_msg = []
-    for i, title in failed_ids:
-        if i in no_format_ids:
-            msg = title + ' (No formats)'
-        else:
-            msg = title + ' (ISBN not found)'
-        det_msg.append(msg)
-    if same_isbn_ids:
-        if det_msg:
-            det_msg.append('----------------------------------')
-        for i, title in same_isbn_ids:
-            msg = title + ' (Same ISBN)'
-            det_msg.append(msg)
-    if len(extracted_ids) > 0:
-        if det_msg:
-            det_msg.append('----------------------------------')
-        for i, title, last_modified, isbn in extracted_ids:
-            msg = '%s (Extracted %s)'%(title, isbn)
-            det_msg.append(msg)
-
-    det_msg = '\n'.join(det_msg)
-    return extracted_ids, same_isbn_ids, failed_ids, det_msg
-
 def get_url_list(urls):
     def f(x):
         if x.strip(): return True

defaults.ini

@@ -36,6 +36,7 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
+language_label:Language
 characters_label:Characters
 series_label:Series
 ## Completed/In-Progress
@@ -67,7 +68,7 @@ version_label:FFDL Version
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -128,22 +129,55 @@ extratags: FanFiction
 ## Primarily for commandline.
 #slow_down_sleep_time:0.5
 
-## output background color--only used by html and epub (and ignored in
-## epub by many readers).  Must be hex code, # will be added.
-background_color: ffffff
-
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
 #post_process_cmd: addbook -f "${output_filename}" -t "${title}"
 
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
 
 ## Each output format has a section that overrides [defaults]
 [html]
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css:
+ body { background-color: #%(background_color)s; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -163,11 +197,44 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css: 
+ body { background-color: #%(background_color)s;
+        text-align: justify;
+        margin: 2%%; }
+ pre { font-size: x-small; }
+ sml { font-size: small; }
+ h1 { text-align: center; }
+ h2 { text-align: center; }
+ h3 { text-align: center; }
+ h4 { text-align: center; }
+ h5 { text-align: center; }
+ h6 { text-align: center; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
-
 ## Each site has a section that overrides [defaults] *and* the format
 ## sections test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses

downloader.py

@@ -46,7 +46,6 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     return output_filename
 
 def main():
-
    # read in args, anything starting with -- will be treated as --<varible>=<value>
    usage = "usage: %prog [options] storyurl"
    parser = OptionParser(usage)
@@ -215,4 +214,7 @@ def main():
        print us
    
 if __name__ == "__main__":
+    #import time
+    #start = time.time()
     main()
+    #print("Total time seconds:%f"%(time.time()-start))

fanficdownloader/adapters/__init__.py

@@ -44,6 +44,7 @@ import adapter_twiwritenet
 import adapter_whoficcom
 import adapter_siyecouk
 import adapter_archiveofourownorg
+import adapter_ficbooknet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -242,21 +242,20 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             if chapnotes != None:
                 chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
                 chapter.append(chapnotes)
-        
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        chapfoot = soup.find('div', {'class' : "end notes module"})
 		
-        soup = soup.find('div', {'class' : "userstuff module"})
-        chtext = soup.find('h3', {'class' : "landmark heading"})
+        text = soup.find('div', {'class' : "userstuff module"})
+        chtext = text.find('h3', {'class' : "landmark heading"})
         if chtext:
             chtext.extract()
-        chapter.append(soup)
+        chapter.append(text)
 		
+        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
         if chapfoot != None:
             chapfoot = chapfoot.find('blockquote')
             chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
             chapter.append(chapfoot)
 		
+        footnotes = soup.find('div', {'id' : "work_endnotes"})
         if footnotes != None:
             footnotes = footnotes.find('blockquote')
             chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -83,8 +83,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             
         if "Unable to locate story with id of " in data:
             raise exceptions.StoryDoesNotExist(url)
-            
-        if "Chapter not found. Please check to see you are not using an outdated url." in data:
+
+        # some times "Chapter not found...", sometimes "Chapter text not found..."
+        if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
         try:
@@ -102,7 +103,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                                            chapcount+1)
             print('=Trying newer chapter: %s' % tryurl)
             newdata = self._fetchUrl(tryurl)
-            if "Chapter not found. Please check to see you are not using an outdated url." \
+            if "not found. Please check to see you are not using an outdated url." \
                     not in newdata:
                 print('=======Found newer chapter: %s' % tryurl)
                 soup = bs.BeautifulSoup(newdata)
@@ -201,7 +202,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         metatext = a.findNext(text=re.compile(r' - Reviews:'))
         if metatext == None: # indicates there's no Reviews, look for id: instead.
             metatext = a.findNext(text=re.compile(r' - id:'))
-        #print("========= metatext:\n%s"%metatext)
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
@@ -215,7 +215,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
         # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
         # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?:[ ]+with characters (?P<char1>.*?\.?)(?: & (?P<char2>.*?\.?))?\. )?",
+        # <meta name="description" content="Chapter 1 of a Harry Potter and Transformers  - Humor/Adventure crossover fanfiction  with characters: Harry P. & Ironhide. IT’s one thing to be tossed thru the Veil for something he didn’t do. It was quite another to wake in his animigus form in a world not his own. Harry just knew someone was laughing at him somewhere. Mech/Mech pairings inside..">
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
                      soup.find('meta',{'name':'description'})['content'])
         if m != None:
             genres=m.group('genres')
@@ -225,7 +226,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 for g in genres.split('/'):
                     self.story.addToList('genre',g)
 
-            if m.group('char1') != None:
+            if m.group('chars') != None:
+
                 # At this point we've proven that there's character(s)
                 # We can't reliably parse characters out of meta name="description".
                 # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
@@ -233,12 +235,16 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 # reviewstext can take form of:
                 # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
                 # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
+                # - English - Humor/Adventure -  Harry P. & Ironhide - Reviews:
                 mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
                               metatext)
                 chars = mc.group("chars")
                 for c in chars.split(' & '):
                     self.story.addToList('characters',c)
-        
+        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
+        if m.group('lang') != None:
+            self.story.setMetadata('language',m.group('lang'))
+                    
         return
 
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import datetime
+import logging
+import re
+import urllib2
+from .. import translit
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+
+def getClass():
+    return FicBookNetAdapter
+
+
+class FicBookNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fbn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %m %Y" 
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ficbook.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/readfic/12345"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        url=self.url
+        logging.debug("URL: "+url)
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+				
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+		
+        table = soup.find('td',{'width':'50%'})
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
+        
+        # Find authorid and URL from... author url.
+        a = table.find('a')
+        self.story.setMetadata('authorId',a.text) # Author's name is unique
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.text)
+        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
+
+        # Find the chapters:
+        chapters = soup.find('div', {'class' : 'part_list'})
+        if chapters != None:
+            chapters=chapters.findAll('a', href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+"/\d+#part_content$"))
+            self.story.setMetadata('numChapters',len(chapters))
+            for x in range(0,len(chapters)):
+                chapter=chapters[x]
+                churl='http://'+self.host+chapter['href']
+                self.chapterUrls.append((stripHTML(chapter),churl))
+                if x == 0:
+                    pubdate = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+                if x == len(chapters)-1:
+                    update = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            self.story.setMetadata('numChapters',1)
+            pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
+            update=pubdate
+
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+
+        if not ',' in pubdate:
+            pubdate=datetime.date.today().strftime(self.dateformat)
+        if not ',' in update:
+            update=datetime.date.today().strftime(self.dateformat)
+        pubdate=pubdate.split(',')[0]
+        update=update.split(',')[0]
+		
+        fullmon = {"yanvarya":"01", "января":"01",
+           "fievralya":"02", "февраля":"02",
+           "marta":"03", "марта":"03",
+           "aprielya":"04", "апреля":"04",
+           "maya":"05", "мая":"05",
+           "iyunya":"06", "июня":"06",
+           "iyulya":"07", "июля":"07",
+           "avghusta":"08", "августа":"08",
+           "sentyabrya":"09", "сентября":"09",
+           "oktyabrya":"10", "октября":"10",
+           "noyabrya":"11", "ноября":"11",
+           "diekabrya":"12", "декабря":"12" }
+        for (name,num) in fullmon.items():
+            if name in pubdate:
+                pubdate = pubdate.replace(name,num)
+            if name in update:
+                update = update.replace(name,num)
+
+        self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
+        self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
+        self.story.setMetadata('language','Russian')
+		
+        pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
+        pr='http://'+self.host+pr['href']
+        pr = bs.BeautifulSoup(self._fetchUrl(pr))
+        pr=pr.findAll('div', {'class' : 'part_text'})
+        i=0
+        for part in pr:
+            i=i+len(stripHTML(part).split(' '))
+        self.story.setMetadata('numWords', str(i))
+		
+        i=0
+        fandoms = table.findAll('a', href=re.compile(r'/fanfiction/\w+'))
+        for fandom in fandoms:
+            self.story.addToList('category',fandom.string)
+            i=i+1
+        if i > 1:
+            self.story.addToList('genre', 'Кроссовер')
+		
+        meta=table.findAll('a', href=re.compile(r'/ratings/'))
+        i=0
+        for m in meta:
+            if i == 0:
+                self.story.setMetadata('rating', m.find('b').text)
+                i=1
+            elif i == 1:
+                if not "," in m.nextSibling:
+                    i=2
+                self.story.addToList('genre', m.find('b').text)
+            elif i == 2:
+                self.story.addToList('warnings', m.find('b').text)
+		
+
+        if table.find('span', {'style' : 'color: green'}):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In Progress')
+		
+
+        tags = table.findAll('b')
+        for tag in tags:
+            label = translit.translit(tag.text)
+            if 'Piersonazhi:' in label or 'Персонажи:' in label:
+                chars=tag.nextSibling.string.split(', ')
+                for char in chars:
+                    self.story.addToList('characters',char)
+                break
+				
+        summary=soup.find('span', {'class' : 'urlize'})
+        self.story.setMetadata('description', summary.text)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        chapter = soup.find('div', {'class' : 'public_beta'})
+        if chapter == None:
+            chapter = soup.find('div', {'class' : 'public_beta_disabled'})
+
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return utf8FromSoup(chapter)

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -203,6 +203,10 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 	# our div with poor html inside the story text.
 	data = data.replace('<!-- headerend -->','<crazytagstringnobodywouldstumbleonaccidently id="storytext">').replace('<!-- footerstart -->','</crazytagstringnobodywouldstumbleonaccidently>')
         
+        # problems with some stories confusing Soup.  This is a nasty
+        # hack, but it works.
+        data = data[data.index("<crazytagstringnobodywouldstumbleonaccidently"):]
+
         soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         body = soup.findAll('body') ## some stories use a nested body and body

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -201,9 +201,6 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet tends to fail more if hit too fast.
-                        ## This is in additional to what ever the
-                        ## slow_down_sleep_time setting is.
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -20,6 +20,7 @@ import logging
 import re
 import urllib2
 import cookielib as cl
+import datetime
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
@@ -142,18 +143,27 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             pass
         self.story.setMetadata('description', description_soup.text)
         
-        # Unfortunately, nowhere on the page is the year mentioned. Because we would much rather update the story needlessly
-        # than miss an update, we hardcode the year of creation and update to be 2011.
+        # Unfortunately, nowhere on the page is the year mentioned.
+        # Best effort to deal with this:
+        # Use this year, if that's a date in the future, subtract one year.
+        # Their earliest story is Jun, so they'll probably change the date
+        # around then.
 
+        now = datetime.datetime.now()
+        
         # Get the date of creation from the first chapter
         datePublished_text = chapterDates[0]
         day, month = datePublished_text.split()
         day = re.sub(r"[^\d.]+", '', day)
-        datePublished = makeDate("2011"+month+day, "%Y%b%d")
+        datePublished = makeDate("%s%s%s"%(now.year,month,day), "%Y%b%d")
+        if datePublished > now :
+            datePublished = datePublished.replace(year=now.year-1)
         self.story.setMetadata("datePublished", datePublished)
         dateUpdated_soup = bs.BeautifulSoup(data).find("div", {"class":"calendar"})
         dateUpdated_soup.find('span').extract()
-        dateUpdated = makeDate("2011"+dateUpdated_soup.text, "%Y%b%d")
+        dateUpdated = makeDate("%s%s"%(now.year,dateUpdated_soup.text), "%Y%b%d")
+        if dateUpdated > now :
+            dateUpdated = datePublished.replace(year=now.year-1)
         self.story.setMetadata("dateUpdated", dateUpdated)
         
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_test1.py

@@ -90,6 +90,16 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
             self.story.setMetadata('status','In-Progress')
         else:
             self.story.setMetadata('status','Completed')
+
+        langs = {
+            0:"English",
+            1:"Russian",
+            2:"French",
+            3:"German",
+            }
+        if idnum < 10:
+            self.story.setMetadata('language',langs[idnum%len(langs)])
+        # greater than 10, no language.
             
         self.setSeries('The Great Test',idnum)
         
@@ -104,7 +114,8 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('category','Harry Potter')
         self.story.addToList('category','Furbie')
         self.story.addToList('category','Crossover')
-        
+        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','SF')
         self.story.addToList('genre','Noir')
@@ -116,7 +127,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
                             ('Chapter 6',self.url+"&chapter=6"),
-                            # ('Chapter 7',self.url+"&chapter=6"),
+                            ('Chapter 7',self.url+"&chapter=6"),
                             # ('Chapter 8',self.url+"&chapter=6"),
                             # ('Chapter 9',self.url+"&chapter=6"),
                             # ('Chapter 0',self.url+"&chapter=6"),

fanficdownloader/adapters/base_adapter.py

@@ -40,6 +40,7 @@ except:
     #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
 
 from ..story import Story
+from ..gziphttp import GZipProcessor
 from ..configurable import Configurable
 from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
 from ..exceptions import InvalidStoryURL
@@ -72,7 +73,7 @@ class BaseSiteAdapter(Configurable):
         self.password = ""
         self.is_adult=False
         
-        self.opener = u2.build_opener(u2.HTTPCookieProcessor())
+        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         self.storyDone = False
         self.metadataDone = False
         self.story = Story()

fanficdownloader/gziphttp.py

@@ -0,0 +1,38 @@
+## Borrowed from http://techknack.net/python-urllib2-handlers/
+
+import urllib2
+from gzip import GzipFile
+from StringIO import StringIO
+
+class GZipProcessor(urllib2.BaseHandler):
+    """A handler to add gzip capabilities to urllib2 requests
+    """
+    def http_request(self, req):
+        req.add_header("Accept-Encoding", "gzip")
+        return req
+    https_request = http_request
+
+    def http_response(self, req, resp):
+        #print("Content-Encoding:%s"%resp.headers.get("Content-Encoding"))
+        if resp.headers.get("Content-Encoding") == "gzip":
+            gz = GzipFile(
+                        fileobj=StringIO(resp.read()),
+                        mode="r"
+                      )
+#            resp.read = gz.read
+#            resp.readlines = gz.readlines
+#            resp.readline = gz.readline
+#            resp.next = gz.next
+            old_resp = resp
+            resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        return resp
+    https_response = http_response
+
+# brave new world - 1:30 w/o, 1:10 with?  40 chapters, so 20s from sleeps.
+# with gzip, no sleep: 47.469
+# w/o gzip, no sleep: 47.736
+
+# I Am What I Am 67 chapters
+# w/o gzip: 57.168
+# w/ gzip: 40.692

fanficdownloader/htmlcleanup.py

@@ -27,7 +27,7 @@ def _unirepl(match):
     return unichr(value)
 
 def _replaceNumberEntities(data):
-    p = re.compile(r'&#(x?)(\d+);')
+    p = re.compile(r'&#(x?)([0-9a-fA-F]+);')
     return p.sub(_unirepl, data)
 
 def _replaceNotEntities(data):

fanficdownloader/story.py

@@ -15,10 +15,54 @@
 # limitations under the License.
 #
 
-import os
+import os, re
 
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
+# The list comes from ffnet, the only multi-language site we support
+# at the time of writing.  Values are taken largely from pycountry,
+# but with some corrections and guesses.
+langs = {
+    "English":"en",
+    "Spanish":"es",
+    "French":"fr",
+    "German":"de",
+    "Chinese":"zh",
+    "Japanese":"ja",
+    "Dutch":"nl",
+    "Portuguese":"pt",
+    "Russian":"ru",
+    "Italian":"it",
+    "Bulgarian":"bg",
+    "Polish":"pl",
+    "Hungarian":"hu",
+    "Hebrew":"he",
+    "Arabic":"ar",
+    "Swedish":"sv",
+    "Norwegian":"no",
+    "Danish":"da",
+    "Finnish":"fi",
+    "Filipino":"fil",
+    "Esperanto":"eo",
+    "Hindi":"hi",
+    "Punjabi":"pa",
+    "Farsi":"fa",
+    "Greek":"el",
+    "Romanian":"ro",
+    "Albanian":"sq",
+    "Serbian":"sr",
+    "Turkish":"tr",
+    "Czech":"cs",
+    "Indonesian":"id",
+    "Croatian":"hr",
+    "Catalan":"ca",
+    "Latin":"la",
+    "Korean":"ko",
+    "Vietnamese":"vi",
+    "Thai":"th",
+    "Devanagari":"hi",
+    }
+
 class Story:
     
     def __init__(self):
@@ -26,16 +70,28 @@ class Story:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
             self.metadata = {'version':'4.3'}
+        self.replacements = []
         self.chapters = [] # chapters will be tuples of (title,html)
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
 
     def setMetadata(self, key, value):
         ## still keeps < < and &
         self.metadata[key]=conditionalRemoveEntities(value)
+        if key == "language":
+            try:
+                self.metadata['langcode'] = langs[self.metadata[key]]
+            except:
+                self.metadata['langcode'] = 'en'
 
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
             return self.metadata[key]
+
+    def doReplacments(self,value):
+        for (p,v) in self.replacements:
+            if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
+                value = re.sub(p,v,value)                
+        return value;
         
     def getMetadata(self, key, removeallentities=False):
         value = None
@@ -50,7 +106,8 @@ class Story:
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
                  value = value.strftime("%Y-%m-%d")
-                 
+
+        value=self.doReplacments(value)
         if removeallentities and value != None:
             return removeAllEntities(value)
         else:
@@ -81,10 +138,14 @@ class Story:
     def getList(self,listname):
         if not self.listables.has_key(listname):
             return []
-        return self.listables[listname]
+        return filter( lambda x : x!=None and x!='' ,
+                       map(self.doReplacments,self.listables[listname]) )
 
     def getLists(self):
-        return self.listables
+        lsts = {}
+        for ln in self.listables.keys():
+            lsts[ln] = self.getList(ln)
+        return lsts
     
     def addChapter(self, title, html):
         self.chapters.append( (title,html) )
@@ -96,6 +157,11 @@ class Story:
     def __str__(self):
         return "Metadata: " +str(self.metadata) + "\nListables: " +str(self.listables) #+ "\nChapters: "+str(self.chapters)
 
+    def setReplace(self,replace):
+        for line in replace.splitlines():
+            if "=>" in line:
+                self.replacements.append(map( lambda x: x.strip(), line.split("=>") ))
+    
 def commaGroups(s):
     groups = []
     while s and s[-1].isdigit():

fanficdownloader/translit.py

@@ -0,0 +1,57 @@
+#-*-coding:utf-8-*-
+# Code taken from http://python.su/forum/viewtopic.php?pid=66946
+import unicodedata
+def is_syllable(letter):
+    syllables = ("A", "E", "I", "O", "U", "a", "e", "i", "o", "u")
+    if letter in syllables:
+        return True
+    return False
+def is_consonant(letter):
+    return not is_syllable(letter)
+def romanize(letter):
+    try:
+        str(letter)
+    except UnicodeEncodeError:
+        pass
+    else:
+        return str(letter)
+    unid = unicodedata.name(letter)
+    exceptions = {"NUMERO SIGN": "No", "LEFT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "DASH": "-"}
+    for name_contains in exceptions:
+        if unid.find(name_contains)!=-1:
+            return exceptions[name_contains]
+    assert(unid.startswith("CYRILLIC"))# Not ready to romanize anything but cyrillics
+    transformation_pairs = {"CYRILLIC CAPITAL LETTER ": str.capitalize, "CYRILLIC SMALL LETTER ": str.lower}
+    func = str.lower
+    for name_contains in transformation_pairs:
+        if unid.find(name_contains)!=-1:
+            func = transformation_pairs[name_contains]
+            unid = unid.replace(name_contains, "")
+    cyrillic_exceptions = {"YERU": "y", "SHORT I": "y", "HARD SIGN": "\'", "SOFT SIGN": "\'", "BYELORUSSIAN-UKRAINIAN I": "i", "GHE WITH UPTURN": "g", "UKRAINIAN IE": "ie", "YU": "yu", "YA": "ya"}
+    for name_contains in cyrillic_exceptions:
+        if unid.find(name_contains)!=-1:
+            return cyrillic_exceptions[name_contains]
+    if all(map(is_syllable, unid)):
+        return func(unid)
+    else:
+        return func(filter(is_consonant, unid))
+def translit(text):
+    output = ""
+    for letter in text:
+        output += romanize(letter)
+    return output
+#def main():
+    #text = u"русск.: Любя, съешь щипцы, — вздохнёт мэр, — кайф жгуч."
+    #print translit(text)
+    #text = u"укр.: Гей, хлопці, не вспію - на ґанку ваша файна їжа знищується бурундучком."
+    #print translit(text)
+    #text = u"болг.: Ах, чудна българска земьо, полюшквай цъфтящи жита."
+    #print translit(text)
+    #text = u"серб.: Неуредне ноћне даме досађивале су Џеку К."
+    #print translit(text)
+    #russk.: Lyubya, s'iesh' shchiptsy, - vzdohniot mer, - kayf zhghuch.
+    #ukr.: Ghiey, hloptsi, nie vspiyu - na ganku vasha fayna yzha znishchuiet'sya burunduchkom.
+    #bolgh.: Ah, chudna b'lgharska ziem'o, polyushkvay ts'ftyashchi zhita.
+    #sierb.: Nieuriednie notshnie damie dosadjivalie su Dzhieku K.
+if __name__=="__main__":
+    main()

fanficdownloader/writers/base_writer.py

@@ -46,9 +46,13 @@ class BaseStoryWriter(Configurable):
         
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
+        
+        self.story.setReplace(self.getConfig('replace_metadata'))
+        
         self.validEntries = [
             'category',
             'genre',
+            'language',
             'characters',
             'series',
             'status',
@@ -77,6 +81,7 @@ class BaseStoryWriter(Configurable):
         self.titleLabels = {
             'category':'Category',
             'genre':'Genre',
+            'language':'Language',
             'status':'Status',
             'series':'Series',
             'characters':'Characters',
@@ -194,6 +199,12 @@ class BaseStoryWriter(Configurable):
         if outfilename == None:
             outfilename=self.getOutputFileName()
 
+        # minor cheat, tucking css into metadata.
+        if self.getConfig("output_css"):
+            self.story.metadata["output_css"] = self.getConfig("output_css")
+        else:
+            self.story.metadata["output_css"] = ''
+            
         if not outstream:
             close=True
             logging.debug("Save directly to file: %s" % outfilename)

fanficdownloader/writers/writer_epub.py

@@ -41,32 +41,7 @@ class EpubWriter(BaseStoryWriter):
     def __init__(self, config, story):
         BaseStoryWriter.__init__(self, config, story)
 
-        self.EPUB_CSS = string.Template('''
-body { margin: 2%;
-       text-align: justify;
-       background-color: #${background_color}; }
-pre { font-size: x-small; }
-sml { font-size: small; }
-h1 { text-align: center; }
-h2 { text-align: center; }
-h3 { text-align: center; }
-h4 { text-align: center; }
-h5 { text-align: center; }
-h6 { text-align: center; }
-.CI {
-    text-align:center;
-    margin-top:0px;
-    margin-bottom:0px;
-    padding:0px;
-    }
-.center   {text-align: center;}
-.cover    {text-align: center;}
-.full     {width: 100%; }
-.quarter  {width: 25%; }
-.smcap    {font-variant: small-caps;}
-.u        {text-decoration: underline;}
-.bold     {font-weight: bold;}
-''')
+        self.EPUB_CSS = string.Template('''${output_css}''')
         
         self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -228,7 +203,10 @@ h6 { text-align: center; }
 
         metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
         metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
-        metadata.appendChild(newTag(contentdom,"dc:language",text="en"))
+        if self.story.getMetadata('langcode') != None:
+            metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
+        else:
+            metadata.appendChild(newTag(contentdom,"dc:language",text='en'))
 
         #  published, created, updated, calibre
         #  Leave calling self.story.getMetadataRaw directly in case date format changes.
@@ -361,7 +339,7 @@ h6 { text-align: center; }
         del tocncxdom
 
         # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")})) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 
 
         # write title page.
         if self.getConfig("titlepage_use_table"):
@@ -424,4 +402,4 @@ def newTag(dom,name,attrs=None,text=None):
     if( text is not None ):
         tag.appendChild(dom.createTextNode(text))
     return tag
-    
+

fanficdownloader/writers/writer_html.py

@@ -39,20 +39,7 @@ class HTMLWriter(BaseStoryWriter):
 <head>
 <title>${title} by ${author}</title>
 <style type="text/css">
-body { background-color: #${background_color}; }
-.CI {
-    text-align:center;
-    margin-top:0px;
-    margin-bottom:0px;
-    padding:0px;
-    }
-.center   {text-align: center;}
-.cover    {text-align: center;}
-.full     {width: 100%; }
-.quarter  {width: 25%; }
-.smcap    {font-variant: small-caps;}
-.u        {text-decoration: underline;}
-.bold     {font-weight: bold;}
+${output_css}
 </style>
 </head>
 <body>
@@ -95,9 +82,6 @@ body { background-color: #${background_color}; }
 
     def writeStoryImpl(self, out):
 
-        # minor cheat, tucking bg into metadata.
-        if self.getConfig("background_color"):
-            self.story.metadata["background_color"] = self.getConfig("background_color")
         self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
 
         self.writeTitlePage(out,

fanficdownloader/writers/writer_mobi.py

@@ -41,7 +41,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -64,7 +63,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -91,7 +89,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <div>
@@ -113,7 +110,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${chapter}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3>${chapter}</h3>

index.html

@@ -54,34 +54,30 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>Support for 'Series'</h3>
+	  <h3>New Russian Language Site ficbook.net</h3>
 	  <p>
-	    We now collect 'Series' name and number for the sites:
-	      harrypotterfanfiction.com,
-	      potionsandsnitches.net,
-	      adastrafanfic.com,
-	      whofic.com,
-	      fanfiction.tenhawkpresents.com,
-	      castlefans.org,
-	      tthfanfic.org,
-	      www.siye.co.uk,
-	      twilighted.net*,
-	      twilighted.net* and 
-	      thewriterscoffeeshop.com*.
+	    Thanks to Ida Leter's hard work, we now support <a href="http://ficbook.net">ficbook.net</a>, a Russian language fanfiction site.
+	  </p>
+	  <h3>Support for Language, Custom CSS and Replacement of Metadata</h3>
+	  <p>
+	    There's now a 'Language' metadata field that can be filled, if the site supports different languages.  Currently, it's only used
+	    with fanfiction.net and ficbook.net.
 	  </p>
 	  <p>
-	    * The last three use series as reading lists and stories collections as much as true story series,
-	    so they default to <i>not</i> collect series info.  You can turn it on in your User Configuration if you want.
+	    The CSS included in the HTML and EPUB output formats is now a customizable parameter.
 	  </p>
-	  <h3>New Site: <a href="http://archiveofourown.org">archiveofourown.org</a></h3>
 	  <p>
-	    Thanks to Ida Leter for writing the code to support a new site: <a href="http://archiveofourown.org">archiveofourown.org</a>.
+	    There's now a customizable parameter to include a list of regular expressions to replace metadata as you see fit.
+	  </p>
+	  <p>
+	    Examples of how to use both new features can be found in the
+	    <a href="http://www.mobileread.com/forums/showthread.php?p=1962034#post1962034">plugin forum</a>.
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-3-0.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-3-1.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -251,9 +247,16 @@
 	    <br /><a href="http://archiveofourown.org/works/76366/chapters/101584">http://archiveofourown.org/works/76366/chapters/101584</a>.
 	  </dd>
 	</dl>	
-
-	
-	A few additional things to know, which will make your life substantially easier:
+	  <dt>ficbook.net(Russian)</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://ficbook.net/readfic/93626">http://ficbook.net/readfic/93626</a>.
+	    <br /><a href="http://ficbook.net/readfic/93626/246417#part_content">http://ficbook.net/readfic/93626/246417#part_content</a>.
+	  </dd>
+	</dl>
+	<p>
+	  A few additional things to know, which will make your life substantially easier:
+	</p>
 	<ol>
 	  <li>
 	    First thing to know: I do not use your Google login and password. In fact, all I know about it is your ID &ndash; password

plugin-defaults.ini

@@ -41,6 +41,7 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
+language_label:Language
 characters_label:Characters
 series_label:Series
 ## Completed/In-Progress
@@ -72,7 +73,7 @@ version_label:FFDL Version
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -111,9 +112,54 @@ extratags: FanFiction
 ## epub by many readers).  Must be hex code, # will be added.
 background_color: ffffff
 
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css:
+ body { background-color: #%(background_color)s; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
+[txt]
+## Add URLs since there aren't links.
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+
+## use \r\n for line endings, the windows convention.  text output only.
+windows_eol: true
+
 [txt]
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
@@ -133,6 +179,40 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css: 
+ body { background-color: #%(background_color)s;
+        text-align: justify;
+        margin: 2%%; }
+ pre { font-size: x-small; }
+ sml { font-size: small; }
+ h1 { text-align: center; }
+ h2 { text-align: center; }
+ h3 { text-align: center; }
+ h4 { text-align: center; }
+ h5 { text-align: center; }
+ h6 { text-align: center; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true