Bump vers for AO3 adapter, category->genre in fimfic, adult check for AO3.

& plugin.  Save plugin bump to 1.3.1.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-3-2
+version: 4-3-1
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 4, 6)
+    version             = (1, 3, 2)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import traceback, copy
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont,
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
                       QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant)
 
 from calibre.gui2 import dynamic, info_dialog
@@ -58,7 +58,10 @@ copylist = ['personal.ini',
             'updatedefault',
             'fileform',
             'collision',
-            'deleteotherforms']
+            'deleteotherforms',
+            'addtolists',
+            'addtoreadlists',
+            'addtolistsonread']
 
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
@@ -290,11 +293,6 @@ class PersonalIniTab(QWidget):
         self.l.addWidget(self.label)
 
         self.ini = QTextEdit(self)
-        try:
-            self.ini.setFont(QFont("Courier",
-                                   self.plugin_action.gui.font().pointSize()+1));
-        except Exception as e:
-            print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
@@ -326,11 +324,6 @@ class ShowDefaultsIniDialog(QDialog):
         
         self.ini = QTextEdit(self)
         self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
-        try:
-            self.ini.setFont(QFont("Courier",
-                                   get_gui().font().pointSize()+1));
-        except Exception as e:
-            print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(text)
         self.ini.setReadOnly(True)
@@ -435,9 +428,7 @@ class OtherTab(QWidget):
                                                   and dynamic[key] is False:
                 dynamic[key] = True
         info_dialog(self, _('Done'),
-                    _('Confirmation dialogs have all been reset'),
-                    show=True,
-                    show_copy_button=False)
+                _('Confirmation dialogs have all been reset'), show=True)
 
 permitted_values = {
     'int' : ['numWords','numChapters'],
@@ -447,7 +438,6 @@ permitted_values = {
     'series' : ['series'],
     'enumeration' : ['category',
                      'genre',
-                     'language',
                      'series',
                      'characters',
                      'status',
@@ -480,7 +470,6 @@ permitted_values['comments'] = permitted_values['enumeration']
 titleLabels = {
     'category':'Category',
     'genre':'Genre',
-    'language':'Language',
     'status':'Status',
     'status-C':'Status:Completed',
     'status-I':'Status:In-Progress',

calibre-plugin/dialogs.py

@@ -60,8 +60,9 @@ class DroppableQTextEdit(QTextEdit):
             return QTextEdit.canInsertFromMimeData(self,source)
 
     def insertFromMimeData(self, source):
-        if source.hasText():
-            self.append(source.text())
+        if source.hasUrls():
+            for u in source.urls():
+                self.append(u.toString())
         else:
             return QTextEdit.insertFromMimeData(self, source)
                             

calibre-plugin/ffdl_plugin.py

@@ -52,6 +52,9 @@ formmapping = {
 
 PLUGIN_ICONS = ['images/icon.png']
 
+sendlists = ["Send to Nook", "Send to Kindle", "Send to Droid", "Add to Nook", "Add to Kindle", "Add to Droid"]
+readlists = ["000"]
+
 class FanFictionDownLoaderPlugin(InterfaceAction):
 
     name = 'FanFictionDownLoader'
@@ -116,10 +119,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def about_to_show_menu(self):
         self.rebuild_menus()
 
-    def library_changed(self, db):
-        # We need to reset our menus after switching libraries
-        self.rebuild_menus()
-        
     def rebuild_menus(self):
         with self.menus_lock:
             # Show the config dialog
@@ -240,35 +239,22 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
     def get_list_urls(self):
         if len(self.gui.library_view.get_selected_ids()) > 0:
-            book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
+            url_list = []
+            for book_id in self.gui.library_view.get_selected_ids():
+                url = self._get_story_url(self.gui.current_db, book_id)
+                if url != None:
+                    url_list.append(url)
 
-            LoopProgressDialog(self.gui,
-                               book_list,
-                               partial(self._get_story_url_for_list, db=self.gui.current_db),
-                               self._finish_get_list_urls,
-                               init_label="Collecting URLs for stories...",
-                               win_title="Get URLs for stories",
-                               status_prefix="URL retrieved")
+            if url_list:
+                d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
+                d.setWindowIcon(get_icon('bookmarks.png'))
+                d.exec_()
+            else:
+                info_dialog(self.gui, _('List of URLs'),
+                            _('No URLs found in selected books.'),
+                            show=True,
+                            show_copy_button=False)
             
-    def _get_story_url_for_list(self,book,db=None):
-        book['url'] = self._get_story_url(db,book['calibre_id'])
-        if book['url'] == None:
-            book['good']=False
-        else:
-            book['good']=True
-            
-    def _finish_get_list_urls(self, book_list):
-        url_list = [ x['url'] for x in book_list if x['good'] ]
-        if url_list:
-            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
-            d.setWindowIcon(get_icon('bookmarks.png'))
-            d.exec_()
-        else:
-            info_dialog(self.gui, _('List of URLs'),
-                        _('No URLs found in selected books.'),
-                        show=True,
-                        show_copy_button=False)
-                
     def add_dialog(self):
 
         #print("add_dialog()")
@@ -304,29 +290,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if len(self.gui.library_view.get_selected_ids()) == 0:
             return
         #print("update_existing()")
-        
+        previous = self.gui.library_view.currentIndex()
         db = self.gui.current_db
-        book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
-        #book_ids = self.gui.library_view.get_selected_ids()
-
-        LoopProgressDialog(self.gui,
-                           book_list,
-                           partial(self._populate_book_from_calibre_id, db=self.gui.current_db),
-                           self._update_existing_2,
-                           init_label="Collecting stories for update...",
-                           win_title="Get stories for updates",
-                           status_prefix="URL retrieved")            
-        
-        #books = self._convert_calibre_ids_to_books(db, book_ids)
+        book_ids = self.gui.library_view.get_selected_ids()
+        books = self._convert_calibre_ids_to_books(db, book_ids)
         #print("update books:%s"%books)
 
-    def _update_existing_2(self,book_list):
-        
         d = UpdateExistingDialog(self.gui,
                                  'Update Existing List',
                                  prefs,
                                  self.qaction.icon(),
-                                 book_list,
+                                 books,
                                  )
         d.exec_()
         if d.result() != d.Accepted:
@@ -365,11 +339,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
         
-        if 0 < len(filter(lambda x : x['good'], books)):
-            LoopProgressDialog(self.gui,
-                               books,
-                               partial(self.get_metadata_for_book, options = options),
-                               partial(self.start_download_list, options = options))
+        LoopProgressDialog(self.gui,
+                           books,
+                           partial(self.get_metadata_for_book, options = options),
+                           partial(self.start_download_list, options = options))
         # LoopProgressDialog calls get_metadata_for_book for each 'good' story,
         # get_metadata_for_book updates book for each,
         # LoopProgressDialog calls start_download_list at the end which goes
@@ -685,15 +658,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             total_good = len(good_list)
 
             self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
-
-            if total_good > 0:
-                LoopProgressDialog(self.gui,
-                                   good_list,
-                                   partial(self._update_book, options=options, db=self.gui.current_db),
-                                   partial(self._update_books_completed, options=options),
-                                   init_label="Updating calibre for stories...",
-                                   win_title="Update calibre for stories",
-                                   status_prefix="Updated")
+            
+            LoopProgressDialog(self.gui,
+                               good_list,
+                               partial(self._update_book, options=options, db=self.gui.current_db),
+                               partial(self._update_books_completed, options=options),
+                               init_label="Updating calibre for stories...",
+                               win_title="Update calibre for stories",
+                               status_prefix="Updated")
 
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -739,16 +711,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
                 old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
             # mi.tags needs to be list, but set kills dups.
-            mi.tags = list(set(list(old_tags)+mi.tags))
-
-        if 'langcode' in book['all_metadata']:
-            mi.languages=[book['all_metadata']['langcode']]
-        else:
-            # Set language english, but only if not already set.
-            oldmi = db.get_metadata(book_id,index_is_id=True)
-            if not oldmi.languages:
-                mi.languages=['eng']
-                
+            mi.tags = list(set(list(old_tags)+mi.tags)) 
+        # Set language english, but only if not already set.
+        oldmi = db.get_metadata(book_id,index_is_id=True)
+        if not oldmi.languages:
+            mi.languages=['eng']
         db.set_metadata(book_id,mi)
 
         # do configured column updates here.
@@ -887,30 +854,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         self._set_book_url_and_comment(book,url)
         return book
         
-    def _convert_id_to_book(self, idval, good=True):
-        book = {}
-        book['good'] = good
-        book['calibre_id'] = idval
-        book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
-            
-        book['comment'] = ''
-        book['url'] = ''
-        book['added'] = False
-        
-        return book
-        
     
-    # def _convert_calibre_ids_to_books(self, db, ids):
-    #     books = []
-    #     for book_id in ids:
-    #         books.append(self._convert_calibre_id_to_book(db,book_id))
-    #     return books
+    def _convert_calibre_ids_to_books(self, db, ids):
+        books = []
+        for book_id in ids:
+            books.append(self._convert_calibre_id_to_book(db,book_id))
+        return books
             
-    def _populate_book_from_calibre_id(self, book, db=None):
-        mi = db.get_metadata(book['calibre_id'], index_is_id=True)
-        #book = {}
+    def _convert_calibre_id_to_book(self, db, book_id):
+        mi = db.get_metadata(book_id, index_is_id=True)
+        book = {}
         book['good'] = True
         book['calibre_id'] = mi.id
         book['title'] = mi.title
@@ -920,9 +873,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['url'] = ""
         book['added'] = False
         
-        url = self._get_story_url(db,book['calibre_id'])
+        url = self._get_story_url(db,book_id)
         self._set_book_url_and_comment(book,url)
-        #return book
+
+        return book
 
     def _set_book_url_and_comment(self,book,url):
         if not url:
@@ -973,6 +927,40 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         except:
             return None;
 
+
+
+def get_job_details(job):
+    '''
+    Convert the job result into a set of parameters including a detail message
+    summarising the success of the extraction operation.
+    This is used by both the threaded and worker approaches to extraction
+    '''
+    extracted_ids, same_isbn_ids, failed_ids, no_format_ids = job.result
+    if not hasattr(job, 'html_details'):
+        job.html_details = job.details
+    det_msg = []
+    for i, title in failed_ids:
+        if i in no_format_ids:
+            msg = title + ' (No formats)'
+        else:
+            msg = title + ' (ISBN not found)'
+        det_msg.append(msg)
+    if same_isbn_ids:
+        if det_msg:
+            det_msg.append('----------------------------------')
+        for i, title in same_isbn_ids:
+            msg = title + ' (Same ISBN)'
+            det_msg.append(msg)
+    if len(extracted_ids) > 0:
+        if det_msg:
+            det_msg.append('----------------------------------')
+        for i, title, last_modified, isbn in extracted_ids:
+            msg = '%s (Extracted %s)'%(title, isbn)
+            det_msg.append(msg)
+
+    det_msg = '\n'.join(det_msg)
+    return extracted_ids, same_isbn_ids, failed_ids, det_msg
+
 def get_url_list(urls):
     def f(x):
         if x.strip(): return True

defaults.ini

@@ -36,7 +36,6 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
-language_label:Language
 characters_label:Characters
 series_label:Series
 ## Completed/In-Progress
@@ -68,7 +67,7 @@ version_label:FFDL Version
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -129,55 +128,22 @@ extratags: FanFiction
 ## Primarily for commandline.
 #slow_down_sleep_time:0.5
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Must be hex code, # will be added.
+background_color: ffffff
+
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
 #post_process_cmd: addbook -f "${output_filename}" -t "${title}"
 
-## Use regular expressions to find and replace (or remove) metadata.
-## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
-## etc.  See http://docs.python.org/library/re.html (look for re.sub)
-## for regexp details.
-## Make sure to keep at least one space at the start of each line and
-## to escape % to %%, if used.
-#replace_metadata:
-# Sci-Fi=>SF
-# Puella Magi Madoka Magica.* => Madoka
-# Comedy=>Humor
-# Crossover: (.*)=>\1
-# (.*)Great(.*)=>\1Moderate\2
-# .*-Centered=>
 
 ## Each output format has a section that overrides [defaults]
 [html]
 
-## output background color--only used by html and epub (and ignored in
-## epub by many readers).  Included below in output_css--will be
-## ignored if not in output_css.
-background_color: ffffff
-
-## Allow customization of CSS.  Make sure to keep at least one space
-## at the start of each line and to escape % to %%.  Also need
-## background_color to be in the same section, if included in CSS.
-output_css:
- body { background-color: #%(background_color)s; }
- .CI {
-     text-align:center;
-     margin-top:0px;
-     margin-bottom:0px;
-     padding:0px;
-     }
- .center   {text-align: center;}
- .cover    {text-align: center;}
- .full     {width: 100%%; }
- .quarter  {width: 25%%; }
- .smcap    {font-variant: small-caps;}
- .u        {text-decoration: underline;}
- .bold     {font-weight: bold;}
-
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -197,44 +163,11 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL
 
-## output background color--only used by html and epub (and ignored in
-## epub by many readers).  Included below in output_css--will be
-## ignored if not in output_css.
-background_color: ffffff
-
-## Allow customization of CSS.  Make sure to keep at least one space
-## at the start of each line and to escape % to %%.  Also need
-## background_color to be in the same section, if included in CSS.
-output_css: 
- body { background-color: #%(background_color)s;
-        text-align: justify;
-        margin: 2%%; }
- pre { font-size: x-small; }
- sml { font-size: small; }
- h1 { text-align: center; }
- h2 { text-align: center; }
- h3 { text-align: center; }
- h4 { text-align: center; }
- h5 { text-align: center; }
- h6 { text-align: center; }
- .CI {
-     text-align:center;
-     margin-top:0px;
-     margin-bottom:0px;
-     padding:0px;
-     }
- .center   {text-align: center;}
- .cover    {text-align: center;}
- .full     {width: 100%%; }
- .quarter  {width: 25%%; }
- .smcap    {font-variant: small-caps;}
- .u        {text-decoration: underline;}
- .bold     {font-weight: bold;}
-
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
+
 ## Each site has a section that overrides [defaults] *and* the format
 ## sections test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses

downloader.py

@@ -46,6 +46,7 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     return output_filename
 
 def main():
+
    # read in args, anything starting with -- will be treated as --<varible>=<value>
    usage = "usage: %prog [options] storyurl"
    parser = OptionParser(usage)
@@ -214,7 +215,4 @@ def main():
        print us
    
 if __name__ == "__main__":
-    #import time
-    #start = time.time()
     main()
-    #print("Total time seconds:%f"%(time.time()-start))

fanficdownloader/adapters/__init__.py

@@ -44,8 +44,6 @@ import adapter_twiwritenet
 import adapter_whoficcom
 import adapter_siyecouk
 import adapter_archiveofourownorg
-import adapter_ficbooknet
-import adapter_gayauthorsorg
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -134,13 +134,15 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 		
         a = metasoup.find('dd',{'class':"fandom tags"})
         fandoms = a.findAll('a',{'class':"tag"})
-        for fandom in fandoms:
+        fandomstext = [fandom.string for fandom in fandoms]
+        for fandom in fandomstext:
             self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
         if a != None:
             warnings = a.findAll('a',{'class':"tag"})
-            for warning in warnings:
+            warningstext = [warning.string for warning in warnings]
+            for warning in warningstext:
                 if warning.string == "Author Chose Not To Use Archive Warnings":
                     warning.string = "No Archive Warnings Apply"
                 if warning.string != "No Archive Warnings Apply":
@@ -149,24 +151,28 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         a = metasoup.find('dd',{'class':"freeform tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
-            for genre in genres:
+            genrestext = [genre.string for genre in genres]
+            for genre in genrestext:
                 self.story.addToList('genre',genre.string)
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
-            for genre in genres:
+            genrestext = [genre.string for genre in genres]
+            for genre in genrestext:
                 if genre != "Gen":
                     self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
         if a != None:
             chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
+            charstext = [char.string for char in chars]
+            for char in charstext:
                 self.story.addToList('characters',char.string)
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
             chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
+            charstext = [char.string for char in chars]
+            for char in charstext:
                 self.story.addToList('characters',char.string)
 		
 
@@ -222,37 +228,38 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if headnotes != None:
             headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
             if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
+                chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))
                 chapter.append(headnotes)
         
         chapsumm = soup.find('div', {'id' : "summary"})
         if chapsumm != None:
             chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
+            chapter.append(bs.BeautifulSoup("<b>Summary for the Chapter:</b>"))
             chapter.append(chapsumm)
         chapnotes = soup.find('div', {'id' : "notes"})
         if chapnotes != None:
             chapnotes = chapnotes.find('blockquote')
             if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
                 chapter.append(chapnotes)
+        
+        footnotes = soup.find('div', {'id' : "work_endnotes"})
+        chapfoot = soup.find('div', {'class' : "end notes module"})
 		
-        text = soup.find('div', {'class' : "userstuff module"})
-        chtext = text.find('h3', {'class' : "landmark heading"})
+        soup = soup.find('div', {'class' : "userstuff module"})
+        chtext = soup.find('h3', {'class' : "landmark heading"})
         if chtext:
             chtext.extract()
-        chapter.append(text)
+        chapter.append(soup)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
         if chapfoot != None:
             chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
+            chapter.append(bs.BeautifulSoup("<b>Notes for the Chapter:</b>"))
             chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
         if footnotes != None:
             footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
+            chapter.append(bs.BeautifulSoup("<b>Author's Note:</b>"))
             chapter.append(footnotes)
 			
         if None == soup:

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -74,7 +74,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
-            #print("\n===================\n%s\n===================\n"%data)
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -84,9 +83,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             
         if "Unable to locate story with id of " in data:
             raise exceptions.StoryDoesNotExist(url)
-
-        # some times "Chapter not found...", sometimes "Chapter text not found..."
-        if "not found. Please check to see you are not using an outdated url." in data:
+            
+        if "Chapter not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
         try:
@@ -104,7 +102,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                                            chapcount+1)
             print('=Trying newer chapter: %s' % tryurl)
             newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
+            if "Chapter not found. Please check to see you are not using an outdated url." \
                     not in newdata:
                 print('=======Found newer chapter: %s' % tryurl)
                 soup = bs.BeautifulSoup(newdata)
@@ -142,7 +140,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 continue
             if 'var storyid' in script.string:
                 for line in script.string.split('\n'):
-                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;\r?$",line)
+                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;$",line)
                     if m == None : continue
                     var,value = m.groups()
                     # remove javascript escaping from values.
@@ -203,6 +201,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         metatext = a.findNext(text=re.compile(r' - Reviews:'))
         if metatext == None: # indicates there's no Reviews, look for id: instead.
             metatext = a.findNext(text=re.compile(r' - id:'))
+        #print("========= metatext:\n%s"%metatext)
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
@@ -216,8 +215,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
         # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
         # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
-        # <meta name="description" content="Chapter 1 of a Harry Potter and Transformers  - Humor/Adventure crossover fanfiction  with characters: Harry P. & Ironhide. IT’s one thing to be tossed thru the Veil for something he didn’t do. It was quite another to wake in his animigus form in a world not his own. Harry just knew someone was laughing at him somewhere. Mech/Mech pairings inside..">
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?:[ ]+with characters (?P<char1>.*?\.?)(?: & (?P<char2>.*?\.?))?\. )?",
                      soup.find('meta',{'name':'description'})['content'])
         if m != None:
             genres=m.group('genres')
@@ -227,8 +225,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 for g in genres.split('/'):
                     self.story.addToList('genre',g)
 
-            if m.group('chars') != None:
-
+            if m.group('char1') != None:
                 # At this point we've proven that there's character(s)
                 # We can't reliably parse characters out of meta name="description".
                 # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
@@ -236,16 +233,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 # reviewstext can take form of:
                 # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
                 # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
-                # - English - Humor/Adventure -  Harry P. & Ironhide - Reviews:
                 mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
                               metatext)
                 chars = mc.group("chars")
                 for c in chars.split(' & '):
                     self.story.addToList('characters',c)
-        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
-        if m.group('lang') != None:
-            self.story.setMetadata('language',m.group('lang'))
-                    
+        
         return
 
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -1,221 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import datetime
-import logging
-import re
-import urllib2
-from .. import translit
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
-
-
-def getClass():
-    return FicBookNetAdapter
-
-
-class FicBookNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','fbn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %m %Y" 
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.ficbook.net'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/readfic/12345"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url=self.url
-        logging.debug("URL: "+url)
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-				
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        # Now go hunting for all the meta data and the chapter list.
-		
-        table = soup.find('td',{'width':'50%'})
-        
-        ## Title
-        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
-        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
-        
-        # Find authorid and URL from... author url.
-        a = table.find('a')
-        self.story.setMetadata('authorId',a.text) # Author's name is unique
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.text)
-        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
-
-        # Find the chapters:
-        chapters = soup.find('div', {'class' : 'part_list'})
-        if chapters != None:
-            chapters=chapters.findAll('a', href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+"/\d+#part_content$"))
-            self.story.setMetadata('numChapters',len(chapters))
-            for x in range(0,len(chapters)):
-                chapter=chapters[x]
-                churl='http://'+self.host+chapter['href']
-                self.chapterUrls.append((stripHTML(chapter),churl))
-                if x == 0:
-                    pubdate = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
-                if x == len(chapters)-1:
-                    update = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-            self.story.setMetadata('numChapters',1)
-            pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
-            update=pubdate
-
-        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-
-        if not ',' in pubdate:
-            pubdate=datetime.date.today().strftime(self.dateformat)
-        if not ',' in update:
-            update=datetime.date.today().strftime(self.dateformat)
-        pubdate=pubdate.split(',')[0]
-        update=update.split(',')[0]
-		
-        fullmon = {"yanvarya":"01", "января":"01",
-           "fievralya":"02", "февраля":"02",
-           "marta":"03", "марта":"03",
-           "aprielya":"04", "апреля":"04",
-           "maya":"05", "мая":"05",
-           "iyunya":"06", "июня":"06",
-           "iyulya":"07", "июля":"07",
-           "avghusta":"08", "августа":"08",
-           "sentyabrya":"09", "сентября":"09",
-           "oktyabrya":"10", "октября":"10",
-           "noyabrya":"11", "ноября":"11",
-           "diekabrya":"12", "декабря":"12" }
-        for (name,num) in fullmon.items():
-            if name in pubdate:
-                pubdate = pubdate.replace(name,num)
-            if name in update:
-                update = update.replace(name,num)
-
-        self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
-        self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
-        self.story.setMetadata('language','Russian')
-		
-        pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
-        pr='http://'+self.host+pr['href']
-        pr = bs.BeautifulSoup(self._fetchUrl(pr))
-        pr=pr.findAll('div', {'class' : 'part_text'})
-        i=0
-        for part in pr:
-            i=i+len(stripHTML(part).split(' '))
-        self.story.setMetadata('numWords', str(i))
-		
-        i=0
-        fandoms = table.findAll('a', href=re.compile(r'/fanfiction/\w+'))
-        for fandom in fandoms:
-            self.story.addToList('category',fandom.string)
-            i=i+1
-        if i > 1:
-            self.story.addToList('genre', 'Кроссовер')
-		
-        meta=table.findAll('a', href=re.compile(r'/ratings/'))
-        i=0
-        for m in meta:
-            if i == 0:
-                self.story.setMetadata('rating', m.find('b').text)
-                i=1
-            elif i == 1:
-                if not "," in m.nextSibling:
-                    i=2
-                self.story.addToList('genre', m.find('b').text)
-            elif i == 2:
-                self.story.addToList('warnings', m.find('b').text)
-		
-
-        if table.find('span', {'style' : 'color: green'}):
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In Progress')
-		
-
-        tags = table.findAll('b')
-        for tag in tags:
-            label = translit.translit(tag.text)
-            if 'Piersonazhi:' in label or 'Персонажи:' in label:
-                chars=tag.nextSibling.string.split(', ')
-                for char in chars:
-                    self.story.addToList('characters',char)
-                break
-				
-        summary=soup.find('span', {'class' : 'urlize'})
-        self.story.setMetadata('description', summary.text)
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logging.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        chapter = soup.find('div', {'class' : 'public_beta'})
-        if chapter == None:
-            chapter = soup.find('div', {'class' : 'public_beta_disabled'})
-
-        if None == chapter:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return utf8FromSoup(chapter)

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -203,10 +203,6 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 	# our div with poor html inside the story text.
 	data = data.replace('<!-- headerend -->','<crazytagstringnobodywouldstumbleonaccidently id="storytext">').replace('<!-- footerstart -->','</crazytagstringnobodywouldstumbleonaccidently>')
         
-        # problems with some stories confusing Soup.  This is a nasty
-        # hack, but it works.
-        data = data[data.index("<crazytagstringnobodywouldstumbleonaccidently"):]
-
         soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         body = soup.findAll('body') ## some stories use a nested body and body

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -201,6 +201,9 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5) ## ffnet tends to fail more if hit too fast.
+                        ## This is in additional to what ever the
+                        ## slow_down_sleep_time setting is.
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -20,7 +20,6 @@ import logging
 import re
 import urllib2
 import cookielib as cl
-import datetime
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
@@ -143,27 +142,18 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             pass
         self.story.setMetadata('description', description_soup.text)
         
-        # Unfortunately, nowhere on the page is the year mentioned.
-        # Best effort to deal with this:
-        # Use this year, if that's a date in the future, subtract one year.
-        # Their earliest story is Jun, so they'll probably change the date
-        # around then.
+        # Unfortunately, nowhere on the page is the year mentioned. Because we would much rather update the story needlessly
+        # than miss an update, we hardcode the year of creation and update to be 2011.
 
-        now = datetime.datetime.now()
-        
         # Get the date of creation from the first chapter
         datePublished_text = chapterDates[0]
         day, month = datePublished_text.split()
         day = re.sub(r"[^\d.]+", '', day)
-        datePublished = makeDate("%s%s%s"%(now.year,month,day), "%Y%b%d")
-        if datePublished > now :
-            datePublished = datePublished.replace(year=now.year-1)
+        datePublished = makeDate("2011"+month+day, "%Y%b%d")
         self.story.setMetadata("datePublished", datePublished)
         dateUpdated_soup = bs.BeautifulSoup(data).find("div", {"class":"calendar"})
         dateUpdated_soup.find('span').extract()
-        dateUpdated = makeDate("%s%s"%(now.year,dateUpdated_soup.text), "%Y%b%d")
-        if dateUpdated > now :
-            dateUpdated = datePublished.replace(year=now.year-1)
+        dateUpdated = makeDate("2011"+dateUpdated_soup.text, "%Y%b%d")
         self.story.setMetadata("dateUpdated", dateUpdated)
         
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_gayauthorsorg.py

@@ -1,203 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import datetime
-import logging
-import re
-import urllib2
-from urllib import unquote
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
-
-def getClass():
-    return GayAuthorsAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class GayAuthorsAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-       
-		
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        # unqoute, change '_' and ' ' to '-', downcase, and remove non-[a-z0-9-]
-        authid = unquote(self.parsedUrl.path.split('/',)[2])
-        authid = authid.lower().replace('_','-').replace(' ','-')
-        authid = re.sub(r"[^a-z0-9-]","",authid)
-        
-        self.story.setMetadata('authorId',authid)
-        logging.debug("authorId: (%s)"%self.story.getMetadata('authorId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('authorId') + '/' + self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ga')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.gayauthors.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story/author/storytitle"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story/")+r".*?/\w+.*?$"
-
-    
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-
-        url = self.url
-        logging.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        msoup = soup.find('div', {'class' : 'story'})
-        if msoup == None:
-            msoup = soup.find('div', {'class' : 'story ispinned'})
-        csoup = soup.find('div', {'id' : 'story_chapters'})
-        
-        ## Title
-        a = msoup.find('span', {'class' : 'title'})
-        title=a.find('span', {'itemprop' : 'name'})
-        self.story.setMetadata('title',title.text)
-        try:
-            # Find Series name from series URL.
-            series = a.find('span',{'class':"description"})
-            series_name = series.find('a')
-            series_name.extract()
-            series_index = int(series.text.split(' ')[1])
-            self.setSeries(series_name.text, series_index)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-        
-        # Find authorid and URL from... author url.
-        a = msoup.find('a', href=re.compile(r'/author/'+self.story.getMetadata('authorId')))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',a.text)
-		
-
-        # Find the chapters:
-        spans=csoup.findAll('span', {'class' : 'desc chapter-info'})
-        for span in spans:
-            span.extract()
-        for chapter in csoup.findAll('a'):
-            # just in case there's tags, like <i> in chapter titles.
-            a=chapter['href'].split(self.story.getMetadata('author'))
-            a=a[0]+self.story.getMetadata('authorId')+a[1]
-            self.chapterUrls.append((stripHTML(chapter),a))
-
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-		
-        cats = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_0__category\w+$'))
-        for cat in cats:
-            self.story.addToList('category',cat.text)
-		
-        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_1__genre\w+$'))
-        for genre in genres:
-            self.story.addToList('genre',genre.text)
-        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_2__tag\w+$'))
-        for genre in genres:
-            self.story.addToList('genre',genre.text)
-			
-        status = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_3__status\w+$'))
-        self.story.setMetadata('status',status.text)
-		
-        rating = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_4__rating\w+$'))
-        self.story.setMetadata('rating',rating.text)
-		
-        summary = msoup.find('span', {'itemprop' : 'description'})
-        self.story.setMetadata('description',summary.text)
-	
-
-        stats = msoup.find('dl',{'class':'info'})
-        dt = stats.findAll('dt')
-        dd = stats.findAll('dd')
-        for x in range(0,len(dt)):
-            label = dt[x].text
-            value = dd[x].text
-
-            if 'Words:' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Published:' in label:
-                date=stripHTML(value.split(' - ')[0])
-                if ',' in date:
-                    date=datetime.date.today().strftime(self.dateformat)
-                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-				
-            if 'Updated:' in label:
-                date=stripHTML(value.split(' - ')[0])
-                if ',' in date:
-                    date=datetime.date.today().strftime(self.dateformat)
-                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logging.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'chapter-content'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return utf8FromSoup(div)

fanficdownloader/adapters/adapter_test1.py

@@ -90,16 +90,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
             self.story.setMetadata('status','In-Progress')
         else:
             self.story.setMetadata('status','Completed')
-
-        langs = {
-            0:"English",
-            1:"Russian",
-            2:"French",
-            3:"German",
-            }
-        if idnum < 10:
-            self.story.setMetadata('language',langs[idnum%len(langs)])
-        # greater than 10, no language.
             
         self.setSeries('The Great Test',idnum)
         
@@ -114,8 +104,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('category','Harry Potter')
         self.story.addToList('category','Furbie')
         self.story.addToList('category','Crossover')
-        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
-        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+        
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','SF')
         self.story.addToList('genre','Noir')
@@ -127,7 +116,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
                             ('Chapter 6',self.url+"&chapter=6"),
-                            ('Chapter 7',self.url+"&chapter=6"),
+                            # ('Chapter 7',self.url+"&chapter=6"),
                             # ('Chapter 8',self.url+"&chapter=6"),
                             # ('Chapter 9',self.url+"&chapter=6"),
                             # ('Chapter 0',self.url+"&chapter=6"),

fanficdownloader/adapters/base_adapter.py

@@ -40,7 +40,6 @@ except:
     #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
 
 from ..story import Story
-from ..gziphttp import GZipProcessor
 from ..configurable import Configurable
 from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
 from ..exceptions import InvalidStoryURL
@@ -73,7 +72,7 @@ class BaseSiteAdapter(Configurable):
         self.password = ""
         self.is_adult=False
         
-        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        self.opener = u2.build_opener(u2.HTTPCookieProcessor())
         self.storyDone = False
         self.metadataDone = False
         self.story = Story()

fanficdownloader/gziphttp.py

@@ -1,38 +0,0 @@
-## Borrowed from http://techknack.net/python-urllib2-handlers/
-
-import urllib2
-from gzip import GzipFile
-from StringIO import StringIO
-
-class GZipProcessor(urllib2.BaseHandler):
-    """A handler to add gzip capabilities to urllib2 requests
-    """
-    def http_request(self, req):
-        req.add_header("Accept-Encoding", "gzip")
-        return req
-    https_request = http_request
-
-    def http_response(self, req, resp):
-        #print("Content-Encoding:%s"%resp.headers.get("Content-Encoding"))
-        if resp.headers.get("Content-Encoding") == "gzip":
-            gz = GzipFile(
-                        fileobj=StringIO(resp.read()),
-                        mode="r"
-                      )
-#            resp.read = gz.read
-#            resp.readlines = gz.readlines
-#            resp.readline = gz.readline
-#            resp.next = gz.next
-            old_resp = resp
-            resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
-            resp.msg = old_resp.msg
-        return resp
-    https_response = http_response
-
-# brave new world - 1:30 w/o, 1:10 with?  40 chapters, so 20s from sleeps.
-# with gzip, no sleep: 47.469
-# w/o gzip, no sleep: 47.736
-
-# I Am What I Am 67 chapters
-# w/o gzip: 57.168
-# w/ gzip: 40.692

fanficdownloader/htmlcleanup.py

@@ -27,7 +27,7 @@ def _unirepl(match):
     return unichr(value)
 
 def _replaceNumberEntities(data):
-    p = re.compile(r'&#(x?)([0-9a-fA-F]+);')
+    p = re.compile(r'&#(x?)(\d+);')
     return p.sub(_unirepl, data)
 
 def _replaceNotEntities(data):

fanficdownloader/story.py

@@ -15,54 +15,10 @@
 # limitations under the License.
 #
 
-import os, re
+import os
 
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
-# The list comes from ffnet, the only multi-language site we support
-# at the time of writing.  Values are taken largely from pycountry,
-# but with some corrections and guesses.
-langs = {
-    "English":"en",
-    "Spanish":"es",
-    "French":"fr",
-    "German":"de",
-    "Chinese":"zh",
-    "Japanese":"ja",
-    "Dutch":"nl",
-    "Portuguese":"pt",
-    "Russian":"ru",
-    "Italian":"it",
-    "Bulgarian":"bg",
-    "Polish":"pl",
-    "Hungarian":"hu",
-    "Hebrew":"he",
-    "Arabic":"ar",
-    "Swedish":"sv",
-    "Norwegian":"no",
-    "Danish":"da",
-    "Finnish":"fi",
-    "Filipino":"fil",
-    "Esperanto":"eo",
-    "Hindi":"hi",
-    "Punjabi":"pa",
-    "Farsi":"fa",
-    "Greek":"el",
-    "Romanian":"ro",
-    "Albanian":"sq",
-    "Serbian":"sr",
-    "Turkish":"tr",
-    "Czech":"cs",
-    "Indonesian":"id",
-    "Croatian":"hr",
-    "Catalan":"ca",
-    "Latin":"la",
-    "Korean":"ko",
-    "Vietnamese":"vi",
-    "Thai":"th",
-    "Devanagari":"hi",
-    }
-
 class Story:
     
     def __init__(self):
@@ -70,28 +26,16 @@ class Story:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
             self.metadata = {'version':'4.3'}
-        self.replacements = []
         self.chapters = [] # chapters will be tuples of (title,html)
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
 
     def setMetadata(self, key, value):
         ## still keeps < < and &
         self.metadata[key]=conditionalRemoveEntities(value)
-        if key == "language":
-            try:
-                self.metadata['langcode'] = langs[self.metadata[key]]
-            except:
-                self.metadata['langcode'] = 'en'
 
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
             return self.metadata[key]
-
-    def doReplacments(self,value):
-        for (p,v) in self.replacements:
-            if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
-                value = re.sub(p,v,value)                
-        return value;
         
     def getMetadata(self, key, removeallentities=False):
         value = None
@@ -106,8 +50,7 @@ class Story:
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
                  value = value.strftime("%Y-%m-%d")
-
-        value=self.doReplacments(value)
+                 
         if removeallentities and value != None:
             return removeAllEntities(value)
         else:
@@ -138,14 +81,10 @@ class Story:
     def getList(self,listname):
         if not self.listables.has_key(listname):
             return []
-        return filter( lambda x : x!=None and x!='' ,
-                       map(self.doReplacments,self.listables[listname]) )
+        return self.listables[listname]
 
     def getLists(self):
-        lsts = {}
-        for ln in self.listables.keys():
-            lsts[ln] = self.getList(ln)
-        return lsts
+        return self.listables
     
     def addChapter(self, title, html):
         self.chapters.append( (title,html) )
@@ -157,11 +96,6 @@ class Story:
     def __str__(self):
         return "Metadata: " +str(self.metadata) + "\nListables: " +str(self.listables) #+ "\nChapters: "+str(self.chapters)
 
-    def setReplace(self,replace):
-        for line in replace.splitlines():
-            if "=>" in line:
-                self.replacements.append(map( lambda x: x.strip(), line.split("=>") ))
-    
 def commaGroups(s):
     groups = []
     while s and s[-1].isdigit():

fanficdownloader/translit.py

@@ -1,57 +0,0 @@
-#-*-coding:utf-8-*-
-# Code taken from http://python.su/forum/viewtopic.php?pid=66946
-import unicodedata
-def is_syllable(letter):
-    syllables = ("A", "E", "I", "O", "U", "a", "e", "i", "o", "u")
-    if letter in syllables:
-        return True
-    return False
-def is_consonant(letter):
-    return not is_syllable(letter)
-def romanize(letter):
-    try:
-        str(letter)
-    except UnicodeEncodeError:
-        pass
-    else:
-        return str(letter)
-    unid = unicodedata.name(letter)
-    exceptions = {"NUMERO SIGN": "No", "LEFT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "DASH": "-"}
-    for name_contains in exceptions:
-        if unid.find(name_contains)!=-1:
-            return exceptions[name_contains]
-    assert(unid.startswith("CYRILLIC"))# Not ready to romanize anything but cyrillics
-    transformation_pairs = {"CYRILLIC CAPITAL LETTER ": str.capitalize, "CYRILLIC SMALL LETTER ": str.lower}
-    func = str.lower
-    for name_contains in transformation_pairs:
-        if unid.find(name_contains)!=-1:
-            func = transformation_pairs[name_contains]
-            unid = unid.replace(name_contains, "")
-    cyrillic_exceptions = {"YERU": "y", "SHORT I": "y", "HARD SIGN": "\'", "SOFT SIGN": "\'", "BYELORUSSIAN-UKRAINIAN I": "i", "GHE WITH UPTURN": "g", "UKRAINIAN IE": "ie", "YU": "yu", "YA": "ya"}
-    for name_contains in cyrillic_exceptions:
-        if unid.find(name_contains)!=-1:
-            return cyrillic_exceptions[name_contains]
-    if all(map(is_syllable, unid)):
-        return func(unid)
-    else:
-        return func(filter(is_consonant, unid))
-def translit(text):
-    output = ""
-    for letter in text:
-        output += romanize(letter)
-    return output
-#def main():
-    #text = u"русск.: Любя, съешь щипцы, — вздохнёт мэр, — кайф жгуч."
-    #print translit(text)
-    #text = u"укр.: Гей, хлопці, не вспію - на ґанку ваша файна їжа знищується бурундучком."
-    #print translit(text)
-    #text = u"болг.: Ах, чудна българска земьо, полюшквай цъфтящи жита."
-    #print translit(text)
-    #text = u"серб.: Неуредне ноћне даме досађивале су Џеку К."
-    #print translit(text)
-    #russk.: Lyubya, s'iesh' shchiptsy, - vzdohniot mer, - kayf zhghuch.
-    #ukr.: Ghiey, hloptsi, nie vspiyu - na ganku vasha fayna yzha znishchuiet'sya burunduchkom.
-    #bolgh.: Ah, chudna b'lgharska ziem'o, polyushkvay ts'ftyashchi zhita.
-    #sierb.: Nieuriednie notshnie damie dosadjivalie su Dzhieku K.
-if __name__=="__main__":
-    main()

fanficdownloader/writers/base_writer.py

@@ -46,13 +46,9 @@ class BaseStoryWriter(Configurable):
         
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
-        
-        self.story.setReplace(self.getConfig('replace_metadata'))
-        
         self.validEntries = [
             'category',
             'genre',
-            'language',
             'characters',
             'series',
             'status',
@@ -81,7 +77,6 @@ class BaseStoryWriter(Configurable):
         self.titleLabels = {
             'category':'Category',
             'genre':'Genre',
-            'language':'Language',
             'status':'Status',
             'series':'Series',
             'characters':'Characters',
@@ -199,12 +194,6 @@ class BaseStoryWriter(Configurable):
         if outfilename == None:
             outfilename=self.getOutputFileName()
 
-        # minor cheat, tucking css into metadata.
-        if self.getConfig("output_css"):
-            self.story.metadata["output_css"] = self.getConfig("output_css")
-        else:
-            self.story.metadata["output_css"] = ''
-            
         if not outstream:
             close=True
             logging.debug("Save directly to file: %s" % outfilename)

fanficdownloader/writers/writer_epub.py

@@ -41,7 +41,32 @@ class EpubWriter(BaseStoryWriter):
     def __init__(self, config, story):
         BaseStoryWriter.__init__(self, config, story)
 
-        self.EPUB_CSS = string.Template('''${output_css}''')
+        self.EPUB_CSS = string.Template('''
+body { margin: 2%;
+       text-align: justify;
+       background-color: #${background_color}; }
+pre { font-size: x-small; }
+sml { font-size: small; }
+h1 { text-align: center; }
+h2 { text-align: center; }
+h3 { text-align: center; }
+h4 { text-align: center; }
+h5 { text-align: center; }
+h6 { text-align: center; }
+.CI {
+    text-align:center;
+    margin-top:0px;
+    margin-bottom:0px;
+    padding:0px;
+    }
+.center   {text-align: center;}
+.cover    {text-align: center;}
+.full     {width: 100%; }
+.quarter  {width: 25%; }
+.smcap    {font-variant: small-caps;}
+.u        {text-decoration: underline;}
+.bold     {font-weight: bold;}
+''')
         
         self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -203,10 +228,7 @@ class EpubWriter(BaseStoryWriter):
 
         metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
         metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
-        if self.story.getMetadata('langcode') != None:
-            metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
-        else:
-            metadata.appendChild(newTag(contentdom,"dc:language",text='en'))
+        metadata.appendChild(newTag(contentdom,"dc:language",text="en"))
 
         #  published, created, updated, calibre
         #  Leave calling self.story.getMetadataRaw directly in case date format changes.
@@ -339,7 +361,7 @@ class EpubWriter(BaseStoryWriter):
         del tocncxdom
 
         # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")})) 
 
         # write title page.
         if self.getConfig("titlepage_use_table"):
@@ -402,4 +424,4 @@ def newTag(dom,name,attrs=None,text=None):
     if( text is not None ):
         tag.appendChild(dom.createTextNode(text))
     return tag
-
+    

fanficdownloader/writers/writer_html.py

@@ -39,7 +39,20 @@ class HTMLWriter(BaseStoryWriter):
 <head>
 <title>${title} by ${author}</title>
 <style type="text/css">
-${output_css}
+body { background-color: #${background_color}; }
+.CI {
+    text-align:center;
+    margin-top:0px;
+    margin-bottom:0px;
+    padding:0px;
+    }
+.center   {text-align: center;}
+.cover    {text-align: center;}
+.full     {width: 100%; }
+.quarter  {width: 25%; }
+.smcap    {font-variant: small-caps;}
+.u        {text-decoration: underline;}
+.bold     {font-weight: bold;}
 </style>
 </head>
 <body>
@@ -82,6 +95,9 @@ ${output_css}
 
     def writeStoryImpl(self, out):
 
+        # minor cheat, tucking bg into metadata.
+        if self.getConfig("background_color"):
+            self.story.metadata["background_color"] = self.getConfig("background_color")
         self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
 
         self.writeTitlePage(out,

fanficdownloader/writers/writer_mobi.py

@@ -41,6 +41,7 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
+<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -63,6 +64,7 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
+<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -89,6 +91,7 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
+<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <div>
@@ -110,6 +113,7 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${chapter}</title>
+<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3>${chapter}</h3>

index.html

@@ -54,15 +54,34 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>New Site gayauthors.org</h3>
+	  <h3>Support for 'Series'</h3>
 	  <p>
-	    Thanks to Ida Leter's hard work, we now support <a href="http://www.gayauthors.org">gayauthors.org</a>, a fanfiction site specializing in gay stories.
+	    We now collect 'Series' name and number for the sites:
+	      harrypotterfanfiction.com,
+	      potionsandsnitches.net,
+	      adastrafanfic.com,
+	      whofic.com,
+	      fanfiction.tenhawkpresents.com,
+	      castlefans.org,
+	      tthfanfic.org,
+	      www.siye.co.uk,
+	      twilighted.net*,
+	      twilighted.net* and 
+	      thewriterscoffeeshop.com*.
+	  </p>
+	  <p>
+	    * The last three use series as reading lists and stories collections as much as true story series,
+	    so they default to <i>not</i> collect series info.  You can turn it on in your User Configuration if you want.
+	  </p>
+	  <h3>New Site: <a href="http://archiveofourown.org">archiveofourown.org</a></h3>
+	  <p>
+	    Thanks to Ida Leter for writing the code to support a new site: <a href="http://archiveofourown.org">archiveofourown.org</a>.
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-3-1.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-3-0.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -231,22 +250,10 @@
 	    <br /><a href="http://archiveofourown.org/works/76366">http://archiveofourown.org/works/76366</a>.
 	    <br /><a href="http://archiveofourown.org/works/76366/chapters/101584">http://archiveofourown.org/works/76366/chapters/101584</a>.
 	  </dd>
-	  <dt>ficbook.net(Russian)</dt>
-	  <dd>
-	    Use the URL of the story, or one of it's chapters, such as
-	    <br /><a href="http://ficbook.net/readfic/93626">http://ficbook.net/readfic/93626</a>.
-	    <br /><a href="http://ficbook.net/readfic/93626/246417#part_content">http://ficbook.net/readfic/93626/246417#part_content</a>.
-	  </dd>
-	  <dt>gayauthors.org</dt>
-	  <dd>
-	    Use the URL of the story, or one of it's chapters, such as
-	    <br /><a href="http://www.gayauthors.org/story/mark-arbour/stvincent">http://www.gayauthors.org/story/mark-arbour/stvincent</a>.
-	    <br /><a href="http://www.gayauthors.org/story/Mark%20Arbour/stvincent/7">http://www.gayauthors.org/story/Mark Arbour/stvincent/7</a>.
-	  </dd>
-	</dl>
-	<p>
-	  A few additional things to know, which will make your life substantially easier:
-	</p>
+	</dl>	
+
+	
+	A few additional things to know, which will make your life substantially easier:
 	<ol>
 	  <li>
 	    First thing to know: I do not use your Google login and password. In fact, all I know about it is your ID &ndash; password

plugin-defaults.ini

@@ -41,7 +41,6 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
-language_label:Language
 characters_label:Characters
 series_label:Series
 ## Completed/In-Progress
@@ -73,7 +72,7 @@ version_label:FFDL Version
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -112,54 +111,9 @@ extratags: FanFiction
 ## epub by many readers).  Must be hex code, # will be added.
 background_color: ffffff
 
-## Use regular expressions to find and replace (or remove) metadata.
-## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
-## etc.  See http://docs.python.org/library/re.html (look for re.sub)
-## for regexp details.
-## Make sure to keep at least one space at the start of each line and
-## to escape % to %%, if used.
-#replace_metadata:
-# Sci-Fi=>SF
-# Puella Magi Madoka Magica.* => Madoka
-# Comedy=>Humor
-# Crossover: (.*)=>\1
-# (.*)Great(.*)=>\1Moderate\2
-# .*-Centered=>
-
 ## Each output format has a section that overrides [defaults]
 [html]
 
-## output background color--only used by html and epub (and ignored in
-## epub by many readers).  Included below in output_css--will be
-## ignored if not in output_css.
-background_color: ffffff
-
-## Allow customization of CSS.  Make sure to keep at least one space
-## at the start of each line and to escape % to %%.  Also need
-## background_color to be in the same section, if included in CSS.
-output_css:
- body { background-color: #%(background_color)s; }
- .CI {
-     text-align:center;
-     margin-top:0px;
-     margin-bottom:0px;
-     padding:0px;
-     }
- .center   {text-align: center;}
- .cover    {text-align: center;}
- .full     {width: 100%%; }
- .quarter  {width: 25%%; }
- .smcap    {font-variant: small-caps;}
- .u        {text-decoration: underline;}
- .bold     {font-weight: bold;}
-
-[txt]
-## Add URLs since there aren't links.
-titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
-
-## use \r\n for line endings, the windows convention.  text output only.
-windows_eol: true
-
 [txt]
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
@@ -179,40 +133,6 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL
 
-## output background color--only used by html and epub (and ignored in
-## epub by many readers).  Included below in output_css--will be
-## ignored if not in output_css.
-background_color: ffffff
-
-## Allow customization of CSS.  Make sure to keep at least one space
-## at the start of each line and to escape % to %%.  Also need
-## background_color to be in the same section, if included in CSS.
-output_css: 
- body { background-color: #%(background_color)s;
-        text-align: justify;
-        margin: 2%%; }
- pre { font-size: x-small; }
- sml { font-size: small; }
- h1 { text-align: center; }
- h2 { text-align: center; }
- h3 { text-align: center; }
- h4 { text-align: center; }
- h5 { text-align: center; }
- h6 { text-align: center; }
- .CI {
-     text-align:center;
-     margin-top:0px;
-     margin-bottom:0px;
-     padding:0px;
-     }
- .center   {text-align: center;}
- .cover    {text-align: center;}
- .full     {width: 100%%; }
- .quarter  {width: 25%%; }
- .smcap    {font-variant: small-caps;}
- .u        {text-decoration: underline;}
- .bold     {font-weight: bold;}
-
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true