Don't mark TtH BtVS when Non-Crossover, unless BtVS/Angel Non-Cross.

a div or p around it. Wraps img tags in a div, defaults true.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-2-1
+version: 4-4-2
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 2, 1)
+    version             = (1, 5, 8)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/about.txt

@@ -16,5 +16,5 @@ The official distro channel for this plugin is there: <a href="http://www.mobile
 group</a> for the downloader.  That covers the web application and CLI, too.
 </p>
 
-The source for this plugin is available
-<a href="http://code.google.com/p/fanficdownloader/source/checkout">here</a>.
+The source for this plugin is available at it's 
+<a href="http://code.google.com/p/fanficdownloader">project home</a>.

calibre-plugin/config.py

@@ -9,8 +9,8 @@ __docformat__ = 'restructuredtext en'
 
 import traceback, copy
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget)
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont,
+                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant)
 
 from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
@@ -35,32 +35,39 @@ all_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
 # Set defaults used by all.  Library specific settings continue to
 # take from here.
 all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
+
 all_prefs.defaults['updatemeta'] = True
+all_prefs.defaults['updatecover'] = False
 all_prefs.defaults['keeptags'] = False
 all_prefs.defaults['urlsfromclip'] = True
 all_prefs.defaults['updatedefault'] = True
 all_prefs.defaults['fileform'] = 'epub'
 all_prefs.defaults['collision'] = OVERWRITE
 all_prefs.defaults['deleteotherforms'] = False
+all_prefs.defaults['adddialogstaysontop'] = False
+all_prefs.defaults['includeimages'] = False
+
 all_prefs.defaults['send_lists'] = ''
 all_prefs.defaults['read_lists'] = ''
 all_prefs.defaults['addtolists'] = False
 all_prefs.defaults['addtoreadlists'] = False
 all_prefs.defaults['addtolistsonread'] = False
 
+all_prefs.defaults['custom_cols'] = {}
+
 # The list of settings to copy from all_prefs or the previous library
 # when config is called for the first time on a library.
 copylist = ['personal.ini',
             'updatemeta',
+            'updatecover',
             'keeptags',
             'urlsfromclip',
             'updatedefault',
             'fileform',
             'collision',
             'deleteotherforms',
-            'addtolists',
-            'addtoreadlists',
-            'addtolistsonread']
+            'adddialogstaysontop',
+            'includeimages']
 
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
@@ -132,20 +139,27 @@ class ConfigWidget(QWidget):
         tab_widget.addTab(self.list_tab, 'Reading Lists')
         if 'Reading List' not in plugin_action.gui.iactions:
             self.list_tab.setEnabled(False)
-            
+
+        self.columns_tab = ColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.columns_tab, 'Custom Columns')
+
         self.other_tab = OtherTab(self, plugin_action)
         tab_widget.addTab(self.other_tab, 'Other')
 
+
     def save_settings(self):
 
         # basic
         prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
         prefs['collision'] = unicode(self.basic_tab.collision.currentText())
         prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
+        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
+        prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
+        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
 
         if self.list_tab:
             # lists
@@ -164,6 +178,15 @@ class ConfigWidget(QWidget):
         else:
             # if they've removed everything, reset to default.
             prefs['personal.ini'] = get_resources('plugin-example.ini')
+
+        # Custom Columns tab
+        colsmap = {}
+        for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
+            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            if val != 'none':
+                colsmap[col] = val
+                #print("colsmap[%s]:%s"%(col,colsmap[col]))
+        prefs['custom_cols'] = colsmap
         
     def edit_shortcuts(self):
         self.save_settings()
@@ -183,6 +206,11 @@ class BasicTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
         horz = QHBoxLayout()
         label = QLabel('Default Output &Format:')
         horz.addWidget(label)
@@ -214,10 +242,15 @@ class BasicTab(QWidget):
         self.l.addLayout(horz)
 
         self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update title, author, URL, tags, etc for story in Calibre from web site.')
+        self.updatemeta.setToolTip('Update title, author, URL, tags, custom columns, etc for story in Calibre from web site.')
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
+        self.updatecover = QCheckBox('Update Cover when Updating Metadata?',self)
+        self.updatecover.setToolTip("Update cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
+        self.updatecover.setChecked(prefs['updatecover'])
+        self.l.addWidget(self.updatecover)
+
         self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
         self.keeptags.setToolTip('Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.')
         self.keeptags.setChecked(prefs['keeptags'])
@@ -239,6 +272,17 @@ class BasicTab(QWidget):
         self.deleteotherforms.setChecked(prefs['deleteotherforms'])
         self.l.addWidget(self.deleteotherforms)
         
+        self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
+        self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
+        self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
+        self.l.addWidget(self.adddialogstaysontop)
+
+        # this is a cheat to make it easier for users to realize there's a new include_images features.
+        self.includeimages = QCheckBox("Include images in EPUBs?",self)
+        self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[defaults]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
+        self.includeimages.setChecked(prefs['includeimages'])
+        self.l.addWidget(self.includeimages)
+        
         self.l.insertStretch(-1)
         
     def set_collisions(self):
@@ -265,10 +309,20 @@ class PersonalIniTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        label = QLabel('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set is_adult and user/password for different sites.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
         self.label = QLabel('personal.ini:')
         self.l.addWidget(self.label)
 
         self.ini = QTextEdit(self)
+        try:
+            self.ini.setFont(QFont("Courier",
+                                   self.plugin_action.gui.font().pointSize()+1));
+        except Exception as e:
+            print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
@@ -293,13 +347,18 @@ class ShowDefaultsIniDialog(QDialog):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
         self.label = QLabel("Plugin Defaults (Read-Only)")
-        self.label.setToolTip("These all of the plugin's configurable settings\nand their default settings.")
+        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
         self.setWindowTitle(_('Plugin Defaults'))
         self.setWindowIcon(icon)
         self.l.addWidget(self.label)
         
         self.ini = QTextEdit(self)
-        self.ini.setToolTip("These all of the plugin's configurable settings\nand their default settings.")
+        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        try:
+            self.ini.setFont(QFont("Courier",
+                                   get_gui().font().pointSize()+1));
+        except Exception as e:
+            print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(text)
         self.ini.setReadOnly(True)
@@ -379,6 +438,11 @@ class OtherTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        label = QLabel("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
         keyboard_shortcuts_button = QPushButton('Keyboard shortcuts...', self)
         keyboard_shortcuts_button.setToolTip(_(
                     'Edit the keyboard shortcuts associated with this plugin'))
@@ -399,5 +463,123 @@ class OtherTab(QWidget):
                                                   and dynamic[key] is False:
                 dynamic[key] = True
         info_dialog(self, _('Done'),
-                _('Confirmation dialogs have all been reset'), show=True)
+                    _('Confirmation dialogs have all been reset'),
+                    show=True,
+                    show_copy_button=False)
 
+permitted_values = {
+    'int' : ['numWords','numChapters'],
+    'float' : ['numWords','numChapters'],
+    'bool' : ['status-C','status-I'],
+    'datetime' : ['datePublished', 'dateUpdated', 'dateCreated'],
+    'series' : ['series'],
+    'enumeration' : ['category',
+                     'genre',
+                     'language',
+                     'series',
+                     'characters',
+                     'status',
+                     'datePublished',
+                     'dateUpdated',
+                     'dateCreated',
+                     'rating',
+                     'warnings',
+                     'numChapters',
+                     'numWords',
+                     'site',
+                     'storyId',
+                     'authorId',
+                     'extratags',
+                     'title',
+                     'storyUrl',
+                     'description',
+                     'author',
+                     'authorUrl',
+                     'formatname'
+                     #,'formatext'   # not useful information.
+                     #,'siteabbrev'
+                     #,'version'
+                     ]
+    }
+# no point copying the whole list.
+permitted_values['text'] = permitted_values['enumeration']
+permitted_values['comments'] = permitted_values['enumeration']
+
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'language':'Language',
+    'status':'Status',
+    'status-C':'Status:Completed',
+    'status-I':'Status:In-Progress',
+    'series':'Series',
+    'characters':'Characters',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'FFDL Version'
+    }
+
+class ColumnsTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.custcol_dropdowns = {}
+
+        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
+
+        for key, column in custom_columns.iteritems():
+
+            if column['datatype'] in permitted_values:
+                # print("\n============== %s ===========\n"%key)
+                # for (k,v) in column.iteritems():
+                #     print("column['%s'] => %s"%(k,v))
+                horz = QHBoxLayout()
+                label = QLabel('%s(%s)'%(column['name'],key))
+                label.setToolTip("Update this %s column with..."%column['datatype'])
+                horz.addWidget(label)
+                dropdown = QComboBox(self)
+                dropdown.addItem('',QVariant('none'))
+                for md in permitted_values[column['datatype']]:
+                    dropdown.addItem(titleLabels[md],QVariant(md))
+                self.custcol_dropdowns[key] = dropdown
+                if key in prefs['custom_cols']:
+                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                if column['datatype'] == 'enumeration':
+                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
+                else:
+                    dropdown.setToolTip("Metadata values valid for this type of column.")
+
+                horz.addWidget(dropdown)
+                self.l.addLayout(horz)
+        
+        self.l.insertStretch(-1)
+
+        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])

calibre-plugin/dcsource.py

@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
-__docformat__ = 'restructuredtext en'
-
-from zipfile import ZipFile
-
-from xml.dom.minidom import parseString
-
-def get_dcsource(inputio):
-    epub = ZipFile(inputio, 'r')
-
-    ## Find the .opf file.
-    container = epub.read("META-INF/container.xml")
-    containerdom = parseString(container)
-    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
-    rootfilename = rootfilenodelist[0].getAttribute("full-path")
-
-    metadom = parseString(epub.read(rootfilename))
-    firstmetadom = metadom.getElementsByTagName("metadata")[0]
-    try:
-        source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8")
-    except:
-        source=None
-
-    return source

calibre-plugin/dialogs.py

@@ -11,13 +11,16 @@ import traceback
 
 from PyQt4 import QtGui
 from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon,
+                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
                       QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt,QAbstractItemView )
+                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView )
 
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 
+from calibre import confirm_config_name
+from calibre.gui2 import dynamic
+
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
@@ -45,13 +48,32 @@ class NotGoingToDownload(Exception):
 
     def __str__(self):
         return self.error
-    
+
+class DroppableQTextEdit(QTextEdit):
+    def __init__(self,parent):
+        QTextEdit.__init__(self,parent)
+        
+    def canInsertFromMimeData(self, source):
+        if source.hasUrls():
+            return True;
+        else:
+            return QTextEdit.canInsertFromMimeData(self,source)
+
+    def insertFromMimeData(self, source):
+        if source.hasText():
+            self.append(source.text())
+        else:
+            return QTextEdit.insertFromMimeData(self, source)
+                            
 class AddNewDialog(SizePersistedDialog):
 
     def __init__(self, gui, prefs, icon, url_list_text):
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
         self.gui = gui
 
+        if prefs['adddialogstaysontop']:
+            QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
+        
         self.setMinimumWidth(300)
         self.l = QVBoxLayout()
         self.setLayout(self.l)
@@ -60,7 +82,7 @@ class AddNewDialog(SizePersistedDialog):
         self.setWindowIcon(icon)
 
         self.l.addWidget(QLabel('Story URL(s), one per line:'))
-        self.url = QTextEdit(self)
+        self.url = DroppableQTextEdit(self)
         self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.')
         self.url.setLineWrapMode(QTextEdit.NoWrap)
         self.url.setText(url_list_text)
@@ -132,30 +154,45 @@ class AddNewDialog(SizePersistedDialog):
         return {
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
-            'updatemeta': unicode(self.updatemeta.isChecked()),
+            'updatemeta': self.updatemeta.isChecked(),
             }
 
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
-        
+
+
+class FakeLineEdit():
+    def __init__(self):
+        pass
+    
+    def text(self):
+        pass
+    
 class UserPassDialog(QDialog):
     '''
     Need to collect User/Pass for some sites.
     '''
-    def __init__(self, gui, site):
+    def __init__(self, gui, site, exception=None):
         QDialog.__init__(self, gui)
         self.gui = gui
         self.status=False
-        self.setWindowTitle('User/Password')
 
         self.l = QGridLayout()
         self.setLayout(self.l)
 
-        self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
+        if exception.passwdonly:
+            self.setWindowTitle('Password')
+            self.l.addWidget(QLabel("Author requires a password for this story(%s)."%exception.url),0,0,1,2)
+            # user isn't used, but it's easier to still have it for
+            # post processing.
+            self.user = FakeLineEdit()
+        else:
+            self.setWindowTitle('User/Password')
+            self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
         
-        self.l.addWidget(QLabel("User:"),1,0)
-        self.user = QLineEdit(self)
-        self.l.addWidget(self.user,1,1)
+            self.l.addWidget(QLabel("User:"),1,0)
+            self.user = QLineEdit(self)
+            self.l.addWidget(self.user,1,1)
    
         self.l.addWidget(QLabel("Password:"),2,0)
         self.passwd = QLineEdit(self)
@@ -180,25 +217,27 @@ class UserPassDialog(QDialog):
         self.status=False
         self.hide()
 
-class MetadataProgressDialog(QProgressDialog):
+class LoopProgressDialog(QProgressDialog):
     '''
     ProgressDialog displayed while fetching metadata for each story.
     '''
     def __init__(self, gui,
                  book_list,
-                 options,
-                 metadata_function,
-                 startdownload_function):
+                 foreach_function,
+                 finish_function,
+                 init_label="Fetching metadata for stories...",
+                 win_title="Downloading metadata for stories",
+                 status_prefix="Fetched metadata for"):
         QProgressDialog.__init__(self,
-                                 "Fetching metadata for stories...",
+                                 init_label,
                                  QString(), 0, len(book_list), gui)
-        self.setWindowTitle("Downloading metadata for stories")
+        self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.gui = gui
         self.book_list = book_list
-        self.options = options
-        self.metadata_function = metadata_function
-        self.startdownload_function = startdownload_function
+        self.foreach_function = foreach_function
+        self.finish_function = finish_function
+        self.status_prefix = status_prefix
         self.i = 0
         
         ## self.do_loop does QTimer.singleShot on self.do_loop also.
@@ -207,7 +246,7 @@ class MetadataProgressDialog(QProgressDialog):
         self.exec_()
 
     def updateStatus(self):
-        self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.book_list)))
+        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
         print(self.labelText())
 
@@ -220,7 +259,7 @@ class MetadataProgressDialog(QProgressDialog):
         try:
             ## collision spec passed into getadapter by partial from ffdl_plugin
             ## no retval only if it exists, but collision is SKIP
-            self.metadata_function(book)
+            self.foreach_function(book)
             
         except NotGoingToDownload as d:
             book['good']=False
@@ -245,7 +284,7 @@ class MetadataProgressDialog(QProgressDialog):
         self.hide()
         self.gui = None        
         # Queues a job to process these books in the background.
-        self.startdownload_function(self.book_list)
+        self.finish_function(self.book_list)
 
 class AboutDialog(QDialog):
 
@@ -290,7 +329,7 @@ class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
 
 class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
-                 save_size_name='FanFictionDownLoader plugin:update list dialog'):
+                 save_size_name='fanfictiondownloader_plugin:update list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
         self.gui = gui
         
@@ -395,17 +434,38 @@ class UpdateExistingDialog(SizePersistedDialog):
         return {
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
-            'updatemeta': unicode(self.updatemeta.isChecked()),
+            'updatemeta': self.updatemeta.isChecked(),
             }
 
+def display_story_list(gui, header, prefs, icon, books,
+                       label_text='',
+                       save_size_name='fanfictiondownloader_plugin:display list dialog',
+                       offer_skip=False):
+    all_good = True
+    for b in books:
+        if not b['good']:
+            all_good=False
+            break
+        
+    ## 
+    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
+        return True
+        pass
+            ## fake accept?
+    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
+                               label_text,
+                               save_size_name,
+                               offer_skip and all_good)
+    d.exec_()
+    return d.result() == d.Accepted
+
 class DisplayStoryListDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
                  label_text='',
-                 save_size_name='FanFictionDownLoader plugin:display list dialog'):
+                 save_size_name='fanfictiondownloader_plugin:display list dialog',
+                 offer_skip=False):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        # UpdateExistingDialog.__init__(self, gui, header, prefs, icon, books,
-        #                               save_size_name='FanFictionDownLoader plugin:display list dialog')
-
+        self.name = save_size_name
         self.gui = gui
         
         self.setWindowTitle(header)
@@ -426,6 +486,15 @@ class DisplayStoryListDialog(SizePersistedDialog):
         #self.label.setWordWrap(True)
         options_layout.addWidget(self.label)
         
+        if offer_skip:
+            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
+            options_layout.addItem(spacerItem1)
+            self.again = QCheckBox('Show this again?',self)
+            self.again.setChecked(True)
+            self.again.stateChanged.connect(self.toggle)
+            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
+            options_layout.addWidget(self.again)
+        
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
@@ -437,10 +506,15 @@ class DisplayStoryListDialog(SizePersistedDialog):
         # Cause our dialog size to be restored from prefs or created on first usage
         self.resize_dialog()
         self.books_table.populate_table(books)
-        
+
     def get_books(self):
         return self.books_table.get_books()
 
+    def toggle(self, *args):
+        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
+
+    
+
 class StoryListTableWidget(QTableWidget):
 
     def __init__(self, parent):

calibre-plugin/ffdl_plugin.py

@@ -15,12 +15,17 @@ from datetime import datetime
 
 from PyQt4.Qt import (QApplication, QMenu, QToolButton)
 
+from PyQt4.Qt import QPixmap, Qt
+from PyQt4.QtCore import QBuffer
+
+
 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation, authors_to_string
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.date import local_tz
 
 # The class that all interface action plugins must inherit from
 from calibre.gui2.actions import InterfaceAction
@@ -29,13 +34,13 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin
                                          create_menu_action_unique, get_library_uuid)
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge
-from calibre_plugins.fanfictiondownloader_plugin.dcsource import get_dcsource
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount
 
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs)
+from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
-    AddNewDialog, UpdateExistingDialog, DisplayStoryListDialog,
-    MetadataProgressDialog, UserPassDialog, AboutDialog,
+    AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
     NotGoingToDownload )
 
@@ -51,9 +56,6 @@ formmapping = {
 
 PLUGIN_ICONS = ['images/icon.png']
 
-sendlists = ["Send to Nook", "Send to Kindle", "Send to Droid", "Add to Nook", "Add to Kindle", "Add to Droid"]
-readlists = ["000"]
-
 class FanFictionDownLoaderPlugin(InterfaceAction):
 
     name = 'FanFictionDownLoader'
@@ -65,7 +67,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     # (text, icon_path, tooltip, keyboard shortcut)
     # icon_path isn't in the zip--icon loaded below.
     action_spec = (name, None,
-                   'Download FanFiction stories from various web sites', None)
+                   'Download FanFiction stories from various web sites', ())
+    # None for keyboard shortcut doesn't allow shortcut.  () does, there just isn't one yet
 
     action_type = 'global'
     # make button menu drop down only
@@ -94,6 +97,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # are not found in the zip file will result in null QIcons.
         icon = get_icon('images/icon.png')
 
+        #self.qaction.setText('FFDL')
+        
         # The qaction is automatically created from the action_spec defined
         # above
         self.qaction.setIcon(icon)
@@ -117,6 +122,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def about_to_show_menu(self):
         self.rebuild_menus()
 
+    def library_changed(self, db):
+        # We need to reset our menus after switching libraries
+        self.rebuild_menus()
+        
     def rebuild_menus(self):
         with self.menus_lock:
             # Show the config dialog
@@ -137,7 +146,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                           triggered=self.update_existing)
 
             if 'Reading List' in self.gui.iactions and (prefs['addtolists'] or prefs['addtoreadlists']) :
-                ## XXX mod and rebuild menu when lists selected/empty
                 self.menu.addSeparator()
                 addmenutxt, rmmenutxt = None, None
                 if prefs['addtolists'] and prefs['addtoreadlists'] :
@@ -164,14 +172,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                                       shortcut_name=rmmenutxt,
                                                                       triggered=partial(self.update_lists,add=False))
                 
-                try:
-                    self.add_send_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-                except:
-                    pass
-                try:
-                    self.add_remove_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-                except:
-                    pass
+                # try:
+                #     self.add_send_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
+                # except:
+                #     pass
+                # try:
+                #     self.add_remove_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
+                # except:
+                #     pass
 
             self.menu.addSeparator()
             self.get_list_action = self.create_menu_item_ex(self.menu, 'Get URLs from Selected Books', image='bookmarks.png',
@@ -192,9 +200,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                            shortcut_name='About FanFictionDownLoader',
                                                            triggered=self.about)
             
-            self.update_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-            self.get_list_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-
             # Before we finalize, make sure we delete any actions for menus that are no longer displayed
             for menu_id, unique_name in self.old_actions_unique_map.iteritems():
                 if menu_id not in self.actions_unique_map:
@@ -215,7 +220,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         text = get_resources('about.txt')
         AboutDialog(self.gui,self.qaction.icon(),self.version + text).exec_()
-        
+
     def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
                            shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
                            unique_name=None):
@@ -234,26 +239,38 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if len(self.gui.library_view.get_selected_ids()) > 0 and \
                 (prefs['addtolists'] or prefs['addtoreadlists']) :
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
-        #self.gui.library_view.model().refresh_ids(self.gui.library_view.get_selected_ids())
 
     def get_list_urls(self):
         if len(self.gui.library_view.get_selected_ids()) > 0:
-            url_list = []
-            for book_id in self.gui.library_view.get_selected_ids():
-                url = self._get_story_url(self.gui.current_db, book_id)
-                if url != None:
-                    url_list.append(url)
+            book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
 
-            if url_list:
-                d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
-                d.setWindowIcon(get_icon('bookmarks.png'))
-                d.exec_()
-            else:
-                info_dialog(self.gui, _('List of URLs'),
-                            _('No URLs found in selected books.'),
-                            show=True,
-                            show_copy_button=False)
+            LoopProgressDialog(self.gui,
+                               book_list,
+                               partial(self._get_story_url_for_list, db=self.gui.current_db),
+                               self._finish_get_list_urls,
+                               init_label="Collecting URLs for stories...",
+                               win_title="Get URLs for stories",
+                               status_prefix="URL retrieved")
             
+    def _get_story_url_for_list(self,book,db=None):
+        book['url'] = self._get_story_url(db,book['calibre_id'])
+        if book['url'] == None:
+            book['good']=False
+        else:
+            book['good']=True
+            
+    def _finish_get_list_urls(self, book_list):
+        url_list = [ x['url'] for x in book_list if x['good'] ]
+        if url_list:
+            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
+            d.setWindowIcon(get_icon('bookmarks.png'))
+            d.exec_()
+        else:
+            info_dialog(self.gui, _('List of URLs'),
+                        _('No URLs found in selected books.'),
+                        show=True,
+                        show_copy_button=False)
+                
     def add_dialog(self):
 
         #print("add_dialog()")
@@ -286,18 +303,32 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         self.start_downloads( options, add_books )
         
     def update_existing(self):
+        if len(self.gui.library_view.get_selected_ids()) == 0:
+            return
         #print("update_existing()")
-        previous = self.gui.library_view.currentIndex()
+        
         db = self.gui.current_db
-        book_ids = self.gui.library_view.get_selected_ids()
-        books = self._convert_calibre_ids_to_books(db, book_ids)
+        book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
+        #book_ids = self.gui.library_view.get_selected_ids()
+
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._populate_book_from_calibre_id, db=self.gui.current_db),
+                           self._update_existing_2,
+                           init_label="Collecting stories for update...",
+                           win_title="Get stories for updates",
+                           status_prefix="URL retrieved")            
+        
+        #books = self._convert_calibre_ids_to_books(db, book_ids)
         #print("update books:%s"%books)
 
+    def _update_existing_2(self,book_list):
+        
         d = UpdateExistingDialog(self.gui,
                                  'Update Existing List',
                                  prefs,
                                  self.qaction.icon(),
-                                 books,
+                                 book_list,
                                  )
         d.exec_()
         if d.result() != d.Accepted:
@@ -336,14 +367,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
         
-        MetadataProgressDialog(self.gui,
+        if 0 < len(filter(lambda x : x['good'], books)):
+            LoopProgressDialog(self.gui,
                                books,
-                               options,
                                partial(self.get_metadata_for_book, options = options),
                                partial(self.start_download_list, options = options))
-        # MetadataProgressDialog calls get_metadata_for_book for each 'good' story,
+        # LoopProgressDialog calls get_metadata_for_book for each 'good' story,
         # get_metadata_for_book updates book for each,
-        # MetadataProgressDialog calls start_download_list at the end which goes
+        # LoopProgressDialog calls start_download_list at the end which goes
         # into the BG, or shows list if no 'good' books.
 
     def get_metadata_for_book(self,book,
@@ -352,7 +383,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                        'updatemeta':True}):
         '''
         Update passed in book dict with metadata from website and
-        necessary data.  To be called from MetadataProgressDialog
+        necessary data.  To be called from LoopProgressDialog
         'loop'.  Also pops dialogs for is adult, user/pass.
         '''
         
@@ -379,18 +410,25 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         ffdlconfig = SafeConfigParser()
         ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
         ffdlconfig.readfp(StringIO(prefs['personal.ini']))
-        adapter = adapters.getAdapter(ffdlconfig,url)
+        adapter = adapters.getAdapter(ffdlconfig,url,fileform)
 
         options['personal.ini'] = prefs['personal.ini']
+        if prefs['includeimages']:
+            # this is a cheat to make it easier for users.
+            options['personal.ini'] = '''[defaults]
+include_images:true
+keep_summary_html:true
+make_firstimage_cover:true
+''' + options['personal.ini']
 
         ## three tries, that's enough if both user/pass & is_adult needed,
         ## or a couple tries of one or the other
         for x in range(0,2):
             try:
                 adapter.getStoryMetadataOnly()
-            except exceptions.FailedToLogin:
+            except exceptions.FailedToLogin, f:
                 print("Login Failed, Need Username/Password.")
-                userpass = UserPassDialog(self.gui,url)
+                userpass = UserPassDialog(self.gui,url,f)
                 userpass.exec_() # exec_ will make it act modal
                 if userpass.status:
                     adapter.username = userpass.user.text()
@@ -406,11 +444,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         story = adapter.getStoryMetadataOnly()
         writer = writers.getWriter(options['fileform'],adapter.config,adapter)
 
+        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
-        book['tags'] = writer.getTags()
-        book['comments'] = story.getMetadata("description") #, removeallentities=True) comments handles entities better.
+        book['tags'] = writer.getTags() # getTags could be moved up into adapter now.  Adapter didn't used to know the fileform
+        book['comments'] = stripHTML(story.getMetadata("description")) #, removeallentities=True) comments handles entities better.
+        book['series'] = story.getMetadata("series")
         
         # adapter.opener is the element with a threadlock.  But del
         # adapter.opener doesn't work--subproc fails when it tries
@@ -422,7 +462,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['password'] = adapter.password
 
         book['icon'] = 'plus.png'
-        book['pubdate'] = story.getMetadataRaw('datePublished')
+        if story.getMetadataRaw('datePublished'):
+            # should only happen when an adapter is broken, but better to
+            # fail gracefully.
+            book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
         book['timestamp'] = None # filled below if not skipped.
         
         if collision in (CALIBREONLY):
@@ -483,13 +526,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 # 'book' can exist without epub.  If there's no existing epub,
                 # let it go and it will download it.
                 if db.has_format(book_id,fileform,index_is_id=True):
-                    toupdateio = StringIO()
-                    (epuburl,chaptercount) = doMerge(toupdateio,
-                                                     [StringIO(db.format(book_id,'EPUB',
-                                                                              index_is_id=True))],
-                                                     titlenavpoints=False,
-                                                     striptitletoc=True,
-                                                     forceunique=False)
+                    (epuburl,chaptercount) = \
+                        get_dcsource_chaptercount(StringIO(db.format(book_id,'EPUB',
+                                                                     index_is_id=True)))
                     urlchaptercount = int(story.getMetadata('numChapters'))
                     if chaptercount == urlchaptercount:
                         if collision == UPDATE:
@@ -523,7 +562,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             # I'm half convinced this should be dateUpdated instead, but
             # this behavior matches how epubs come out when imported
             # dateCreated == packaged--epub/etc created.
-            book['timestamp'] = story.getMetadataRaw('dateCreated')
+            book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
         
         if book['good']: # there shouldn't be any !'good' books at this point.
             # if still 'good', make a temp file to write the output to.
@@ -541,7 +580,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                      'collision':ADDNEW,
                                      'updatemeta':True}):
         '''
-        Called by MetadataProgressDialog to start story downloads BG processing.
+        Called by LoopProgressDialog to start story downloads BG processing.
         adapter_list is a list of tuples of (url,adapter)
         '''
         #print("start_download_list:book_list:%s"%book_list)
@@ -556,18 +595,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.download_list_completed(notjob,options=options)
             return
 
-        # ## XXX show list before starting download.
-        # d = DisplayStoryListDialog(self.gui,
-        #                            'Download List',
-        #                            prefs,
-        #                            self.qaction.icon(),
-        #                            book_list,
-        #                            label_text='Status of stories to be downloaded'
-        #                            )
-        # d.exec_()
-        # if d.result() != d.Accepted:
-        #     return
-            
         for book in book_list:
             if book['good']:
                 break
@@ -596,73 +623,85 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         self.gui.status_bar.show_message('Starting %d FanFictionDownLoads'%len(book_list),3000)
 
+    def _update_book(self,book,db=None,
+                     options={'fileform':'epub',
+                              'collision':ADDNEW,
+                              'updatemeta':True}):
+        print("add/update %s %s"%(book['title'],book['url']))
+        mi = self._make_mi_from_book(book)
+                
+        if options['collision'] != CALIBREONLY:
+            self._add_or_update_book(book,options,prefs,mi)
+
+        if options['collision'] == CALIBREONLY or \
+                (options['updatemeta'] and book['good']):
+            self._update_metadata(db, book['calibre_id'], book, mi, options)
+
+    def _update_books_completed(self, book_list, options={}):
+        
+        add_list = filter(lambda x : x['good'] and x['added'], book_list)
+        update_list = filter(lambda x : x['good'] and not x['added'], book_list)
+        update_ids = [ x['calibre_id'] for x in update_list ]
+        
+        if len(add_list):
+            ## even shows up added to searchs.  Nice.
+            self.gui.library_view.model().books_added(len(add_list))
+
+        if update_ids:
+            self.gui.library_view.model().refresh_ids(update_ids)
+
+        current = self.gui.library_view.currentIndex()
+        self.gui.library_view.model().current_changed(current, self.previous)
+        self.gui.tags_view.recount()
+        
+        if self.gui.cover_flow:
+            self.gui.cover_flow.dataChanged()
+        
+        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
+            
+        if len(update_list) + len(add_list) != len(book_list):
+            d = DisplayStoryListDialog(self.gui,
+                                       'Updates completed, final status',
+                                       prefs,
+                                       self.qaction.icon(),
+                                       book_list,
+                                       label_text='Stories have be added or updated in Calibre, some had additional problems.'
+                                       )
+            d.exec_()
+    
+        print("all done, remove temp dir.")
+        remove_dir(options['tdir'])
+        
     def download_list_completed(self, job, options={}):
         if job.failed:
             self.gui.job_exception(job, dialog_title='Failed to Download Stories')
             return
 
-        previous = self.gui.library_view.currentIndex()
+        self.previous = self.gui.library_view.currentIndex()
         db = self.gui.current_db
 
-        d = DisplayStoryListDialog(self.gui,
-                                   'Downloads finished, confirm to update Calibre',
-                                   prefs,
-                                   self.qaction.icon(),
-                                   job.result,
-                                   label_text='Stories will not be added or updated in Calibre without confirmation.'
-                                   )
-        d.exec_()
-        if d.result() == d.Accepted:
+        if display_story_list(self.gui,
+                              'Downloads finished, confirm to update Calibre',
+                              prefs,
+                              self.qaction.icon(),
+                              job.result,
+                              label_text='Stories will not be added or updated in Calibre without confirmation.',
+                              offer_skip=True):
             
-            ## in case the user removed any from the list.
-            book_list = d.get_books()
-
+            book_list = job.result
             good_list = filter(lambda x : x['good'], book_list)
-
             total_good = len(good_list)
 
-            self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good), 3000)
-            
-            for book in good_list:
-                print("add/update %s %s"%(book['title'],book['url']))
-                mi = self._make_mi_from_book(book)
-                
-                if options['collision'] != CALIBREONLY:
-                    self._add_or_update_book(book,options,prefs,mi)
+            self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
 
-                if options['collision'] == CALIBREONLY or \
-                        (options['updatemeta'] and book['good']) :
-                    self._update_metadata(db, book['calibre_id'], book, mi)
-                    
-            add_list = filter(lambda x : x['good'] and x['added'], book_list)
-            update_list = filter(lambda x : x['good'] and not x['added'], book_list)
-            update_ids = [ x['calibre_id'] for x in update_list ]
-                    
-            if len(add_list):
-                ## even shows up added to searchs.  Nice.
-                self.gui.library_view.model().books_added(len(add_list))
-
-            if update_ids:
-                self.gui.library_view.model().refresh_ids(update_ids)
-
-            current = self.gui.library_view.currentIndex()
-            self.gui.library_view.model().current_changed(current, previous)
-            self.gui.tags_view.recount()
-            
-            self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-            
-            if len(update_list) + len(add_list) != total_good:
-                d = DisplayStoryListDialog(self.gui,
-                                           'Updates completed, final status',
-                                           prefs,
-                                           self.qaction.icon(),
-                                           book_list,
-                                           label_text='Stories have be added or updated in Calibre, some had additional problems.'
-                                           )
-                d.exec_()
-    
-        print("all done, remove temp dir.")
-        remove_dir(options['tdir'])
+            if total_good > 0:
+                LoopProgressDialog(self.gui,
+                                   good_list,
+                                   partial(self._update_book, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options),
+                                   init_label="Updating calibre for stories...",
+                                   win_title="Update calibre for stories",
+                                   status_prefix="Updated")
 
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -698,24 +737,67 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 
         return book_id
 
-    def _update_metadata(self, db, book_id, book, mi):
+    def _update_metadata(self, db, book_id, book, mi, options):
         if prefs['keeptags']:
-            mi = copy.deepcopy(mi)
             old_tags = db.get_tags(book_id)
             # remove old Completed/In-Progress only if there's a new one.
             if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
                 old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
                 # remove old Last Update tags if there are new ones.
-                if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
-                    old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
-                    # mi.tags needs to be list, but set kills dups.
-                mi.tags = list(set(list(old_tags)+mi.tags)) 
-        # Set language english, but only if not already set.
-        oldmi = db.get_metadata(book_id,index_is_id=True)
-        if not oldmi.languages:
-            mi.languages=['eng']
+            if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
+                old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
+            # mi.tags needs to be list, but set kills dups.
+            mi.tags = list(set(list(old_tags)+mi.tags))
+
+        if 'langcode' in book['all_metadata']:
+            mi.languages=[book['all_metadata']['langcode']]
+        else:
+            # Set language english, but only if not already set.
+            oldmi = db.get_metadata(book_id,index_is_id=True)
+            if not oldmi.languages:
+                mi.languages=['eng']
+
+        if options['fileform'] == 'epub' and prefs['updatecover']:
+            existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
+            epubmi = get_metadata(existingepub,'EPUB')
+            if epubmi.cover_data[1] is not None:
+                db.set_cover(book_id, epubmi.cover_data[1])
+            #mi.cover = epubmi.cover_data[1]
+                
         db.set_metadata(book_id,mi)
 
+        # do configured column updates here.
+        #print("all_metadata: %s"%book['all_metadata'])
+        custom_columns = self.gui.library_view.model().custom_columns
+
+        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
+        for col, meta in prefs['custom_cols'].iteritems():
+            #print("setting %s to %s"%(col,meta))
+            if col not in custom_columns:
+                print("%s not an existing column, skipping."%col)
+                continue
+            coldef = custom_columns[col]
+            if not meta.startswith('status-') and meta not in book['all_metadata']:
+                print("No value for %s, skipping."%meta)
+                continue
+            if meta not in permitted_values[coldef['datatype']]:
+                print("%s not a valid column type for %s, skipping."%(col,meta))
+                continue
+            label = coldef['label']
+            if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
+                db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+            elif coldef['datatype'] in ('int','float'):
+                num = unicode(book['all_metadata'][meta]).replace(",","")
+                db.set_custom(book_id, num, label=label, commit=False)
+            elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
+                if meta == 'status-C':
+                    val = book['all_metadata']['status'] == 'Completed'
+                if meta == 'status-I':
+                    val = book['all_metadata']['status'] == 'In-Progress'
+                db.set_custom(book_id, val, label=label, commit=False)
+        
+        db.commit()
+
     def _get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :
             return []
@@ -789,6 +871,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         mi.pubdate = book['pubdate']
         mi.timestamp = book['timestamp']
         mi.comments = book['comments']
+        mi.series = book['series']
         return mi
 
 
@@ -819,35 +902,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         self._set_book_url_and_comment(book,url)
         return book
         
-    
-    def _convert_calibre_ids_to_books(self, db, ids):
-        books = []
-        for book_id in ids:
-            books.append(self._convert_calibre_id_to_book(db,book_id))
-        return books
-            
-    def _convert_calibre_id_to_book(self, db, book_id):
-        mi = db.get_metadata(book_id, index_is_id=True)
+    def _convert_id_to_book(self, idval, good=True):
         book = {}
+        book['good'] = good
+        book['calibre_id'] = idval
+        book['title'] = 'Unknown'
+        book['author'] = 'Unknown'
+        book['author_sort'] = 'Unknown'
+            
+        book['comment'] = ''
+        book['url'] = ''
+        book['added'] = False
+        
+        return book
+            
+    def _populate_book_from_calibre_id(self, book, db=None):
+        mi = db.get_metadata(book['calibre_id'], index_is_id=True)
+        #book = {}
         book['good'] = True
         book['calibre_id'] = mi.id
         book['title'] = mi.title
         book['author'] = authors_to_string(mi.authors)
         book['author_sort'] = mi.author_sort
-        # book['series'] = mi.series
-        # if mi.series:
-        #     book['series_index'] = mi.series_index
-        # else:
-        #     book['series_index'] = 0
-            
         book['comment'] = ''
         book['url'] = ""
         book['added'] = False
         
-        url = self._get_story_url(db,book_id)
+        url = self._get_story_url(db,book['calibre_id'])
         self._set_book_url_and_comment(book,url)
-
-        return book
+        #return book
 
     def _set_book_url_and_comment(self,book,url):
         if not url:
@@ -898,40 +981,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         except:
             return None;
 
-
-
-def get_job_details(job):
-    '''
-    Convert the job result into a set of parameters including a detail message
-    summarising the success of the extraction operation.
-    This is used by both the threaded and worker approaches to extraction
-    '''
-    extracted_ids, same_isbn_ids, failed_ids, no_format_ids = job.result
-    if not hasattr(job, 'html_details'):
-        job.html_details = job.details
-    det_msg = []
-    for i, title in failed_ids:
-        if i in no_format_ids:
-            msg = title + ' (No formats)'
-        else:
-            msg = title + ' (ISBN not found)'
-        det_msg.append(msg)
-    if same_isbn_ids:
-        if det_msg:
-            det_msg.append('----------------------------------')
-        for i, title in same_isbn_ids:
-            msg = title + ' (Same ISBN)'
-            det_msg.append(msg)
-    if len(extracted_ids) > 0:
-        if det_msg:
-            det_msg.append('----------------------------------')
-        for i, title, last_modified, isbn in extracted_ids:
-            msg = '%s (Extracted %s)'%(title, isbn)
-            det_msg.append(msg)
-
-    det_msg = '\n'.join(det_msg)
-    return extracted_ids, same_isbn_ids, failed_ids, det_msg
-
 def get_url_list(urls):
     def f(x):
         if x.strip(): return True

calibre-plugin/jobs.py

@@ -12,10 +12,7 @@ import time, os, traceback
 
 from ConfigParser import SafeConfigParser
 from StringIO import StringIO
-#from itertools import izip
-#from threading import Event
 
-#from calibre.gui2.convert.single import sort_formats_by_preference
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
 from calibre.utils.logging import Log
@@ -23,7 +20,7 @@ from calibre.utils.logging import Log
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.epubmerge import doMerge
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
 
 # ------------------------------------------------------------------------------
 #
@@ -110,7 +107,7 @@ def do_download_for_worker(book,options):
         ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
         ffdlconfig.readfp(StringIO(options['personal.ini']))
         
-        adapter = adapters.getAdapter(ffdlconfig,book['url'])
+        adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
         adapter.is_adult = book['is_adult'] 
         adapter.username = book['username'] 
         adapter.password = book['password']
@@ -135,39 +132,17 @@ def do_download_for_worker(book,options):
         ## checks were done earlier, just update it.
         elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
             
+            # update now handled by pre-populating the old images and
+            # chapters in the adapter rather than merging epubs.
             urlchaptercount = int(story.getMetadata('numChapters'))
-            ## First, get existing epub with titlepage and tocpage stripped.
-            updateio = StringIO()
-            (epuburl,chaptercount) = doMerge(updateio,
-                                             [book['epub_for_update']],
-                                             titlenavpoints=False,
-                                             striptitletoc=True,
-                                             forceunique=False)
+            (url,chaptercount,
+             adapter.oldchapters,
+             adapter.oldimgs) = get_update_data(book['epub_for_update'])
+
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)
 
-            ## Get updated title page/metadata by itself in an epub.
-            ## Even if the title page isn't included, this carries the metadata.
-            titleio = StringIO()
-            writer.writeStory(outstream=titleio,metaonly=True)
-
-            newchaptersio = None
-            if urlchaptercount > chaptercount :
-                ## Go get the new chapters
-                newchaptersio = StringIO()
-                adapter.setChaptersRange(chaptercount+1,urlchaptercount)
-
-                adapter.config.set("overrides",'include_tocpage','false')
-                adapter.config.set("overrides",'include_titlepage','false')
-                writer.writeStory(outstream=newchaptersio)
-
-            ## Merge the three epubs together.
-            doMerge(outfile,
-                    [titleio,updateio,newchaptersio],
-                    fromfirst=True,
-                    titlenavpoints=False,
-                    striptitletoc=False,
-                    forceunique=False)
+            writer.writeStory(outfilename=outfile, forceOverwrite=True)
             
             book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
                 (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)

defaults.ini

@@ -1,4 +1,4 @@
-# Copyright 2011 Fanficdownloader team
+# Copyright 2012 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,7 +36,9 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
+language_label:Language
 characters_label:Characters
+series_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -61,12 +63,19 @@ authorId_label:Author ID
 extratags_label:Extra Tags
 ## The version of fanficdownloader
 ##
-version_label:FFD Version
+version_label:FFDL Version
 
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
 
 ## include title page as first page.
 include_titlepage: true
@@ -117,24 +126,62 @@ extratags: FanFiction
 
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
-## Primarily for commandline.
 #slow_down_sleep_time:0.5
 
-## output background color--only used by html and epub (and ignored in
-## epub by many readers).  Must be hex code, # will be added.
-background_color: ffffff
-
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
 #post_process_cmd: addbook -f "${output_filename}" -t "${title}"
 
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
+
+## Some readers don't show horizontal rule (<hr />) tags correctly.
+## This replaces them all with a centered '* * *'.  (Note centering
+## doesn't work on some devices either.)
+#replace_hr: false
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css:
+ body { background-color: #%(background_color)s; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -154,11 +201,86 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css: 
+ body { background-color: #%(background_color)s;
+        text-align: justify;
+        margin: 2%%; }
+ pre { font-size: x-small; }
+ sml { font-size: small; }
+ h1 { text-align: center; }
+ h2 { text-align: center; }
+ h3 { text-align: center; }
+ h4 { text-align: center; }
+ h5 { text-align: center; }
+ h6 { text-align: center; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
+## include images from img tags in the body and summary of
+## stories.  Images will be converted to jpg for size if possible.
+#include_images:false
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:false
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover: false
+
+## If set, the epub will never have a cover, even include_images is on
+## and the site has specific cover images.
+#never_make_cover: false
+
+## If set, and there isn't already a cover image from the adapter or
+## from make_firstimage_cover, this image will be made the cover.
+## It can be either a 'file:' or 'http:' url.
+## Note that if you enable make_firstimage_cover in [epub], but want
+## to use default_cover_image for a specific site, use the site:format
+## section, for example: [www.ficwad.com:epub]
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/icon.png
+#default_cover_image:http://www.somesite.com/someimage.gif
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+## if the <img> tag doesn't have a div or a p around it, nook gets
+## confused and displays it on every page after that under the text
+## for the rest of the chapter.  I doubt adding a div around the img
+## will break any other readers, but in case it does, the fix can be
+## turned off.
+nook_img_fix:true
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
-
 ## Each site has a section that overrides [defaults] *and* the format
 ## sections test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
@@ -175,8 +297,82 @@ extratags: FanFiction,Testing,Text
 [test1.com:html]
 extratags: FanFiction,Testing,HTML
 
+[castlefans.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.mugglenet.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.tenhawkpresents.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[thequidditchpitch.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.archiveofourown.org]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.fanfiction.net]
 
+[www.ficbook.net]
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fictionally.org storyIds are not unique.  Combine with authorId.
+output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
+
 [www.fictionpress.com]
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
@@ -189,26 +385,37 @@ extratags:
 #username:YourName
 #password:yourpassword
 
-[www.twilighted.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
 
-[www.twiwrite.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
+## fimfiction.net stories can be locked requiring individual
+## passwords.  If fail_on_password is set, the downloader will fail
+## when a password is required rather than prompting every time.
+#fail_on_password: false
 
-[www.whofic.com]
+[www.gayauthors.org]
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.hpfandom.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
 
 [www.mediaminer.org]
 
+[www.potionsandsnitches.net]
+
+[www.siye.co.uk]
+
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -222,7 +429,23 @@ extratags:
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-[www.ficwad.com]
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
+[www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -230,38 +453,21 @@ extratags:
 #username:YourName
 #password:yourpassword
 
-[www.adastrafanfic.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
 
-[www.fictionalley.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
+[www.twiwrite.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
 
-## fictionally.org storyIds are not unique.  Combine with authorId.
-output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
 
-[www.harrypotterfanfiction.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.fimfiction.net]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.tthfanfic.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
+[www.whofic.com]
 
 [overrides]
 ## It may sometimes be useful to override all of the specific format,

downloader.py

@@ -25,19 +25,16 @@ from StringIO import StringIO
 from optparse import OptionParser      
 import getpass
 import string
-
+import ConfigParser
 from subprocess import call
 
-from epubmerge import doMerge
+from fanficdownloader import adapters,writers,exceptions
+from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
     sys.exit(1)
 
-from fanficdownloader import adapters,writers,exceptions
-
-import ConfigParser
-
 def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     writer = writers.getWriter(writeformat,config,adapter)
     writer.writeStory(outstream=outstream,metaonly=metaonly)
@@ -46,7 +43,6 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     return output_filename
 
 def main():
-
    # read in args, anything starting with -- will be treated as --<varible>=<value>
    usage = "usage: %prog [options] storyurl"
    parser = OptionParser(usage)
@@ -67,7 +63,7 @@ def main():
                      help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.  Not compatible with inserted TOC.",)
+                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
                      help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
@@ -117,29 +113,37 @@ def main():
    try:
        ## Attempt to update an existing epub.
        if options.update:
-           updateio = StringIO()
-           (url,chaptercount) = doMerge(updateio,
-                                        args,
-                                        titlenavpoints=False,
-                                        striptitletoc=True,
-                                        forceunique=False)
+           (url,chaptercount) = get_dcsource_chaptercount(args[0])
            print "Updating %s, URL: %s" % (args[0],url)
            output_filename = args[0]
            config.set("overrides","output_filename",args[0])
        else:
            url = args[0]
 
-       adapter = adapters.getAdapter(config,url)
+       adapter = adapters.getAdapter(config,url,options.format)
+
+       ## Check for include_images and absence of PIL, give warning.
+       if adapter.getConfig('include_images'):
+           try:
+               import Image
+           except:
+               print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
+               if not sys.stdin.readline().strip().lower().startswith('y'):
+                   return
+               
 
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
        for x in range(0,2):
            try:
                adapter.getStoryMetadataOnly()
-           except exceptions.FailedToLogin:
-               print "Login Failed, Need Username/Password."
-               sys.stdout.write("Username: ")
-               adapter.username = sys.stdin.readline().strip()
+           except exceptions.FailedToLogin, f:
+               if f.passwdonly:
+                   print "Story requires a password."
+               else:
+                   print "Login Failed, Need Username/Password."
+                   sys.stdout.write("Username: ")
+                   adapter.username = sys.stdin.readline().strip()
                adapter.password = getpass.getpass(prompt='Password: ')
                #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
            except exceptions.AdultCheckRequired:
@@ -156,40 +160,17 @@ def main():
                print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
            else:
                print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
-               ## Get updated title page/metadata by itself in an epub.
-               ## Even if the title page isn't included, this carries the metadata.
-               titleio = StringIO()
-               writeStory(config,adapter,"epub",metaonly=True,outstream=titleio)
-
-               newchaptersio = None
                if not options.metaonly:
-                   ## Go get the new chapters only in another epub.
-                   newchaptersio = StringIO()
-                   adapter.setChaptersRange(chaptercount+1,urlchaptercount)
-                   config.set("overrides",'include_tocpage','false')
-                   config.set("overrides",'include_titlepage','false')
-                   writeStory(config,adapter,"epub",outstream=newchaptersio)
-               
-               # out = open("testing/titleio.epub","wb")
-               # out.write(titleio.getvalue())
-               # out.close()
-               
-               # out = open("testing/updateio.epub","wb")
-               # out.write(updateio.getvalue())
-               # out.close()
-               
-               # out = open("testing/newchaptersio.epub","wb")
-               # out.write(newchaptersio.getvalue())
-               # out.close()
-               
-               ## Merge the three epubs together.
-               doMerge(args[0],
-                       [titleio,updateio,newchaptersio],
-                       fromfirst=True,
-                       titlenavpoints=False,
-                       striptitletoc=False,
-                       forceunique=False)
 
+                   # update now handled by pre-populating the old
+                   # images and chapters in the adapter rather than
+                   # merging epubs.
+                   (url,chaptercount,
+                    adapter.oldchapters,
+                    adapter.oldimgs) = get_update_data(args[0])
+
+                   writeStory(config,adapter,"epub")
+                   
        else:
            # regular download
            if options.metaonly:
@@ -215,4 +196,7 @@ def main():
        print us
    
 if __name__ == "__main__":
+    #import time
+    #start = time.time()
     main()
+    #print("Total time seconds:%f"%(time.time()-start))

editconfig.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -22,7 +22,7 @@
   <body>
     <div id='main' style="width: 80%; margin-left: 10%;">
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
 
       <div style="text-align: center">
@@ -67,7 +67,7 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
       </div>
 

epubmerge.py

@@ -16,374 +16,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import sys
-import os
-import re
-#import StringIO
-from optparse import OptionParser      
-
-import zlib
-import zipfile
-from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
-from time import time
-
-from exceptions import KeyError
-
-from xml.dom.minidom import parse, parseString, getDOMImplementation
-        
-def main(argv):
-    # read in args, anything starting with -- will be treated as --<varible>=<value>
-    usage = "usage: %prog [options] <input epub> [<input epub>...]"
-    parser = OptionParser(usage)
-    parser.add_option("-o", "--output", dest="outputopt", default="merge.epub",
-                      help="Set OUTPUT file, Default: merge.epub", metavar="OUTPUT")
-    parser.add_option("-t", "--title", dest="titleopt", default=None,
-                      help="Use TITLE as the metadata title.  Default: '<first epub title> Anthology'", metavar="TITLE")
-    parser.add_option("-d", "--description", dest="descopt", default=None,
-                      help="Use DESC as the metadata description.  Default: '<epub title> by <author>' for each epub.", metavar="DESC")
-    parser.add_option("-a", "--author",
-                      action="append", dest="authoropts", default=[],
-                      help="Use AUTHOR as a metadata author, multiple authors may be given, Default: <All authors from epubs>", metavar="AUTHOR")
-    parser.add_option("-f", "--first",
-                      action="store_true", dest="fromfirst", default=False,
-                      help="Take all metadata from first input epub",)
-    parser.add_option("-n", "--titles-in-toc",
-                      action="store_true", dest="titlenavpoints",
-                      help="Put an entry in the TOC for each epub, in addition to each epub's chapters.",)
-    parser.add_option("-s", "--strip-title-toc",
-                      action="store_true", dest="striptitletoc",
-                      help="Strip any title_page.xhtml and toc_page.xhtml files.",)
-    
-    (options, args) = parser.parse_args()
-
-    ## Add .epub if not already there.
-    if not options.outputopt.lower().endswith(".epub"):
-        options.outputopt=options.outputopt+".epub"
-
-    print "output file: "+options.outputopt
-    doMerge(options.outputopt,
-            args,
-            options.authoropts,
-            options.titleopt,
-            options.descopt,
-            options.fromfirst,
-            options.titlenavpoints,
-            options.striptitletoc)
-
-    # output = StringIO.StringIO()
-    # files = []
-    # for file in args:
-    #     f = open(file,"rb")
-    #     fio = StringIO.StringIO(f.read())
-    #     f.close()
-    #     files.append(fio)
-
-    # doMerge(output,files,authoropts,titleopt,descopt,fromfirst,titlenavpoints,striptitletoc)
-
-    # out = open(outputopt,"wb")
-    # out.write(output.getvalue())
-    
-def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
-            fromfirst=False,
-            titlenavpoints=True,
-            striptitletoc=False,
-            forceunique=True):
-    '''
-    outputio = output file name or StringIO.
-    files = list of input file names or StringIOs.
-    authoropts = list of authors to use, otherwise add from all input
-    titleopt = title, otherwise '<first title> Anthology'
-    descopt = description, otherwise '<title> by <author>' list for all input
-    fromfirst if true, take all metadata (including author, title, desc) from first input
-    titlenavpoints if true, put in a new TOC entry for each epub
-    striptitletoc if true, strip out any (title|toc)_page.xhtml files
-    forceunique if true, guarantee uniqueness of contents by adding a dir for each input
-    '''
-    ## Python 2.5 ZipFile is rather more primative than later
-    ## versions.  It can operate on a file, or on a StringIO, but
-    ## not on an open stream.  OTOH, I suspect we would have had
-    ## problems with closing and opening again to change the
-    ## compression type anyway.
-
-    filecount=0
-    source=None
-    
-    ## Write mimetype file, must be first and uncompressed.
-    ## Older versions of python(2.4/5) don't allow you to specify
-    ## compression by individual file.
-    ## Overwrite if existing output file.
-    outputepub = ZipFile(outputio, "w", compression=ZIP_STORED)
-    outputepub.debug = 3
-    outputepub.writestr("mimetype", "application/epub+zip")
-    outputepub.close()
-
-    ## Re-open file for content.
-    outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED)
-    outputepub.debug = 3
-
-    ## Create META-INF/container.xml file.  The only thing it does is
-    ## point to content.opf
-    containerdom = getDOMImplementation().createDocument(None, "container", None)
-    containertop = containerdom.documentElement
-    containertop.setAttribute("version","1.0")
-    containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")
-    rootfiles = containerdom.createElement("rootfiles")
-    containertop.appendChild(rootfiles)
-    rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":"content.opf",
-                                                          "media-type":"application/oebps-package+xml"}))
-    outputepub.writestr("META-INF/container.xml",containerdom.toprettyxml(indent='   ',encoding='utf-8'))    
-
-    ## Process input epubs.
-    
-    items = [] # list of (id, href, type) tuples(all strings) -- From .opfs' manifests
-    items.append(("ncx","toc.ncx","application/x-dtbncx+xml")) ## we'll generate the toc.ncx file,
-                                                               ## but it needs to be in the items manifest.
-    itemrefs = [] # list of strings -- idrefs from .opfs' spines
-    navmaps = [] # list of navMap DOM elements -- TOC data for each from toc.ncx files
-
-    booktitles = [] # list of strings -- Each book's title
-    allauthors = [] # list of lists of strings -- Each book's list of authors.
-
-    filelist = []
-    
-    booknum=1
-    firstmetadom = None
-    for file in files:
-        if file == None : continue
-        
-        book = "%d" % booknum
-        bookdir = ""
-        bookid = ""
-        if forceunique:
-            bookdir = "%d/" % booknum
-            bookid = "a%d" % booknum
-        #print "book %d" % booknum
-        
-        epub = ZipFile(file, 'r')
-
-        ## Find the .opf file.
-        container = epub.read("META-INF/container.xml")
-        containerdom = parseString(container)
-        rootfilenodelist = containerdom.getElementsByTagName("rootfile")
-        rootfilename = rootfilenodelist[0].getAttribute("full-path")
-
-        ## Save the path to the .opf file--hrefs inside it are relative to it.
-        relpath = os.path.dirname(rootfilename)
-        if( len(relpath) > 0 ):
-            relpath=relpath+"/"
-            
-        metadom = parseString(epub.read(rootfilename))
-        if booknum==1:
-            firstmetadom = metadom.getElementsByTagName("metadata")[0]
-            try:
-                source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8")
-            except:
-                source=""
-            #print "Source:%s"%source
-
-        ## Save indiv book title
-        booktitles.append(metadom.getElementsByTagName("dc:title")[0].firstChild.data)
-
-        ## Save authors.
-        authors=[]
-        for creator in metadom.getElementsByTagName("dc:creator"):
-            if( creator.getAttribute("opf:role") == "aut" ):
-                authors.append(creator.firstChild.data)
-        allauthors.append(authors)
-
-        for item in metadom.getElementsByTagName("item"):
-            if( item.getAttribute("media-type") == "application/x-dtbncx+xml" ):
-                # TOC file is only one with this type--as far as I know.
-                # grab the whole navmap, deal with it later.
-                tocdom = parseString(epub.read(relpath+item.getAttribute("href")))
-                
-                for navpoint in tocdom.getElementsByTagName("navPoint"):
-                    navpoint.setAttribute("id",bookid+navpoint.getAttribute("id"))
-
-                for content in tocdom.getElementsByTagName("content"):
-                    content.setAttribute("src",bookdir+relpath+content.getAttribute("src"))
-
-                navmaps.append(tocdom.getElementsByTagName("navMap")[0])
-            else:
-                id=bookid+item.getAttribute("id")
-                href=bookdir+relpath+item.getAttribute("href")
-                href=href.encode('utf8')
-                #print "href:"+href
-                if not striptitletoc or not re.match(r'.*/(title|toc)_page\.xhtml',
-                                                      item.getAttribute("href")):
-                    if href not in filelist:
-                        try:
-                            outputepub.writestr(href,
-                                                epub.read(relpath+item.getAttribute("href")))
-                            if re.match(r'.*/(file|chapter)\d+\.x?html',href):
-                                filecount+=1
-                            items.append((id,href,item.getAttribute("media-type")))
-                            filelist.append(href)
-                        except KeyError, ke:
-                            pass # Skip missing files.
-                
-        for itemref in metadom.getElementsByTagName("itemref"):
-
-            if not striptitletoc or not re.match(r'(title|toc)_page', itemref.getAttribute("idref")):
-                itemrefs.append(bookid+itemref.getAttribute("idref"))
-
-        booknum=booknum+1;
-        if not forceunique:
-            # If not forceunique, it's an epub update.
-            # If there's a "calibre_bookmarks.txt", it's from reading
-            # in Calibre and should be preserved.
-            try:
-                fn = "META-INF/calibre_bookmarks.txt"
-                outputepub.writestr(fn,epub.read(fn))
-            except:
-                pass
-        
-
-    ## create content.opf file. 
-    uniqueid="epubmerge-uid-%d" % time() # real sophisticated uid scheme.
-    contentdom = getDOMImplementation().createDocument(None, "package", None)
-    package = contentdom.documentElement
-    if fromfirst and firstmetadom:
-        metadata = firstmetadom
-        firstpackage = firstmetadom.parentNode
-        package.setAttribute("version",firstpackage.getAttribute("version"))
-        package.setAttribute("xmlns",firstpackage.getAttribute("xmlns"))
-        package.setAttribute("unique-identifier",firstpackage.getAttribute("unique-identifier"))
-    else:
-        package.setAttribute("version","2.0")
-        package.setAttribute("xmlns","http://www.idpf.org/2007/opf")
-        package.setAttribute("unique-identifier","epubmerge-id")
-        metadata=newTag(contentdom,"metadata",
-                        attrs={"xmlns:dc":"http://purl.org/dc/elements/1.1/",
-                               "xmlns:opf":"http://www.idpf.org/2007/opf"})
-        metadata.appendChild(newTag(contentdom,"dc:identifier",text=uniqueid,attrs={"id":"epubmerge-id"}))
-        if( titleopt is None ):
-            titleopt = booktitles[0]+" Anthology"
-        metadata.appendChild(newTag(contentdom,"dc:title",text=titleopt))
-    
-        # If cmdline authors, use those instead of those collected from the epubs
-        # (allauthors kept for TOC & description gen below.
-        if( len(authoropts) > 1  ):
-            useauthors=[authoropts]
-        else:
-            useauthors=allauthors
-            
-        usedauthors=dict()
-        for authorlist in useauthors:
-            for author in authorlist:
-                if( not usedauthors.has_key(author) ):
-                    usedauthors[author]=author
-                    metadata.appendChild(newTag(contentdom,"dc:creator",
-                                                attrs={"opf:role":"aut"},
-                                                text=author))
-    
-        metadata.appendChild(newTag(contentdom,"dc:contributor",text="epubmerge",attrs={"opf:role":"bkp"}))
-        metadata.appendChild(newTag(contentdom,"dc:rights",text="Copyrights as per source stories"))
-        metadata.appendChild(newTag(contentdom,"dc:language",text="en"))
-    
-        if not descopt:
-            # created now, but not filled in until TOC generation to save loops.
-            description = newTag(contentdom,"dc:description",text="Anthology containing:\n")
-        else:
-            description = newTag(contentdom,"dc:description",text=descopt)
-        metadata.appendChild(description)
-    
-    package.appendChild(metadata)
-    
-    manifest = contentdom.createElement("manifest")
-    package.appendChild(manifest)
-    for item in items:
-        (id,href,type)=item
-        manifest.appendChild(newTag(contentdom,"item",
-                                       attrs={'id':id,
-                                              'href':href,
-                                              'media-type':type}))
-        
-    spine = newTag(contentdom,"spine",attrs={"toc":"ncx"})
-    package.appendChild(spine)
-    for itemref in itemrefs:
-        spine.appendChild(newTag(contentdom,"itemref",
-                                    attrs={"idref":itemref,
-                                           "linear":"yes"}))
-
-    ## create toc.ncx file
-    tocncxdom = getDOMImplementation().createDocument(None, "ncx", None)
-    ncx = tocncxdom.documentElement
-    ncx.setAttribute("version","2005-1")
-    ncx.setAttribute("xmlns","http://www.daisy.org/z3986/2005/ncx/")
-    head = tocncxdom.createElement("head")
-    ncx.appendChild(head)
-    head.appendChild(newTag(tocncxdom,"meta",
-                            attrs={"name":"dtb:uid", "content":uniqueid}))
-    head.appendChild(newTag(tocncxdom,"meta",
-                            attrs={"name":"dtb:depth", "content":"1"}))
-    head.appendChild(newTag(tocncxdom,"meta",
-                            attrs={"name":"dtb:totalPageCount", "content":"0"}))
-    head.appendChild(newTag(tocncxdom,"meta",
-                            attrs={"name":"dtb:maxPageNumber", "content":"0"}))
-    
-    docTitle = tocncxdom.createElement("docTitle")
-    docTitle.appendChild(newTag(tocncxdom,"text",text=titleopt))
-    ncx.appendChild(docTitle)
-    
-    tocnavMap = tocncxdom.createElement("navMap")
-    ncx.appendChild(tocnavMap)
-
-    ## TOC navPoints can be nested, but this flattens them for
-    ## simplicity, plus adds a navPoint for each epub.
-    booknum=0
-    for navmap in navmaps:
-        navpoints = navmap.getElementsByTagName("navPoint")
-        if titlenavpoints:
-           ## Copy first navPoint of each epub, give a different id and
-           ## text: bookname by authorname
-            newnav = navpoints[0].cloneNode(True)
-            newnav.setAttribute("id","book"+newnav.getAttribute("id"))
-            ## For purposes of TOC titling & desc, use first book author
-            newtext = newTag(tocncxdom,"text",text=booktitles[booknum]+" by "+allauthors[booknum][0])            
-            text = newnav.getElementsByTagName("text")[0]
-            text.parentNode.replaceChild(newtext,text)
-            tocnavMap.appendChild(newnav)
-        
-        if not descopt and not fromfirst:
-            description.appendChild(contentdom.createTextNode(booktitles[booknum]+" by "+allauthors[booknum][0]+"\n"))
-            
-        for navpoint in navpoints:
-            #print "navpoint:%s"%navpoint.getAttribute("id")
-            if not striptitletoc or not re.match(r'(title|toc)_page',navpoint.getAttribute("id")):
-                tocnavMap.appendChild(navpoint)
-        booknum=booknum+1;
-
-    ## Force strict ordering of playOrder
-    playorder=1
-    for navpoint in tocncxdom.getElementsByTagName("navPoint"):
-        navpoint.setAttribute("playOrder","%d" % playorder)
-        if( not navpoint.getAttribute("id").startswith("book") ):
-            playorder = playorder + 1
-
-    ## content.opf written now due to description being filled in
-    ## during TOC generation to save loops.
-    outputepub.writestr("content.opf",contentdom.toxml('utf-8'))
-    outputepub.writestr("toc.ncx",tocncxdom.toxml('utf-8'))
-
-    # declares all the files created by Windows.  otherwise, when
-    # it runs in appengine, windows unzips the files as 000 perms.
-    for zf in outputepub.filelist:
-        zf.create_system = 0
-    outputepub.close()
-
-    return (source,filecount)
-
-## Utility method for creating new tags.
-def newTag(dom,name,attrs=None,text=None):
-    tag = dom.createElement(name)
-    if( attrs is not None ):
-        for attr in attrs.keys():
-            tag.setAttribute(attr,attrs[attr])
-    if( text is not None ):
-        tag.appendChild(dom.createTextNode(text))
-    return tag
     
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    print('''
+The this utility has been split out into it's own project. 
+See: http://code.google.com/p/epubmerge/
+...for a CLI epubmerge.py program and calibre plugin.
+''')

fanficdownloader/adapters/__init__.py

@@ -1,96 +1,103 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob, types
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-from .. import exceptions as exceptions
-
-## must import each adapter here.
-
-import adapter_test1
-import adapter_fanfictionnet
-import adapter_castlefansorg
-import adapter_fanfictionnet
-import adapter_fictionalleyorg
-import adapter_fictionpresscom
-import adapter_ficwadcom
-import adapter_fimfictionnet
-import adapter_harrypotterfanfictioncom
-import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
-import adapter_tenhawkpresentscom
-import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
-import adapter_tthfanficorg
-import adapter_twilightednet
-import adapter_twiwritenet
-import adapter_whoficcom
-import adapter_siyecouk
-
-## This bit of complexity allows adapters to be added by just adding
-## importing.  It eliminates the long if/else clauses we used to need
-## to pick out the adapter.
-    
-## List of registered site adapters.
-__class_list = []
-
-def imports():
-    for name, val in globals().items():
-        if isinstance(val, types.ModuleType):
-            yield val.__name__
-
-for x in imports():
-    if "fanficdownloader.adapters.adapter_" in x:
-        #print x
-        __class_list.append(sys.modules[x].getClass())
-            
-def getAdapter(config,url):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    logging.debug("site:"+domain)
-    cls = getClassFor(domain)
-    if not cls:
-        logging.debug("trying site:www."+domain)
-        cls = getClassFor("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getClassFor(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, re, sys, glob, types
+from os.path import dirname, basename, normpath
+import logging
+import urlparse as up
+
+from .. import exceptions as exceptions
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
+import adapter_archiveofourownorg
+import adapter_ficbooknet
+import adapter_gayauthorsorg
+import adapter_portkeyorg
+import adapter_mugglenetcom
+import adapter_hpfandomnet
+import adapter_thequidditchpitchorg
+
+## This bit of complexity allows adapters to be added by just adding
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
+    
+## List of registered site adapters.
+__class_list = []
+
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        __class_list.append(sys.modules[x].getClass())
+            
+def getAdapter(config,url,fileform=None):
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+    
+    ## remove any trailing '&' parameters--?sid=999 will be left.
+    ## that's all that any of the current adapters need or want.
+    fixedurl = re.sub(r"&.*$","",fixedurl)
+    
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    logging.debug("site:"+domain)
+    cls = getClassFor(domain)
+    if not cls:
+        logging.debug("trying site:www."+domain)
+        cls = getClassFor("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        adapter.setSectionOrder(adapter.getSiteDomain(),fileform)
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getClassFor(domain):
+    for cls in __class_list:
+        if cls.matchesSite(domain):
+            return cls

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
@@ -133,7 +133,8 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                 # sometimes poorly formated desc (<p> w/o </p>) leads
                 # to all labels being included.
                 svalue=svalue[:svalue.find('<span class="label">')]
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -174,13 +175,33 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
             
             if 'Updated' in label:
                 # there's a stray [ at the end.
                 #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
 
     def getChapterText(self, url):
 
@@ -200,7 +221,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)
 
 def getClass():
     return AdAstraFanficComSiteAdapter

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ArchiveOfOurOwnOrgAdapter
+
+
+class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ao3')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%b-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.archiveofourown.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/works/123456"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            addurl = "?view_adult=true"
+        else:
+            addurl=""
+
+        meta = self.url+addurl
+        url = self.url+'/navigate'+addurl
+        logging.debug("URL: "+meta)
+
+        try:
+            data = self._fetchUrl(url)
+            meta = self._fetchUrl(meta)
+
+            if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        metasoup = bs.BeautifulSoup(meta)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r"^/works/\w+"))
+        self.story.setMetadata('title',a.string)
+		
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.text)
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
+        self.story.setMetadata('numChapters',len(chapters))
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        for x in range(0,len(chapters)):
+            # just in case there's tags, like <i> in chapter titles.
+            chapter=chapters[x]
+            if len(chapters)==1:
+                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
+            else:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
+
+
+
+        a = metasoup.find('blockquote',{'class':'userstuff'})
+        if a != None:
+            self.setDescription(url,a.text)
+            #self.story.setMetadata('description',a.text)
+		
+        a = metasoup.find('dd',{'class':"rating tags"})
+        if a != None:
+            self.story.setMetadata('rating',stripHTML(a.text))
+		
+        a = metasoup.find('dd',{'class':"fandom tags"})
+        fandoms = a.findAll('a',{'class':"tag"})
+        for fandom in fandoms:
+            self.story.addToList('category',fandom.string)
+		
+        a = metasoup.find('dd',{'class':"warning tags"})
+        if a != None:
+            warnings = a.findAll('a',{'class':"tag"})
+            for warning in warnings:
+                if warning.string == "Author Chose Not To Use Archive Warnings":
+                    warning.string = "No Archive Warnings Apply"
+                if warning.string != "No Archive Warnings Apply":
+                    self.story.addToList('warnings',warning.string)
+		
+        a = metasoup.find('dd',{'class':"freeform tags"})
+        if a != None:
+            genres = a.findAll('a',{'class':"tag"})
+            for genre in genres:
+                self.story.addToList('genre',genre.string)
+        a = metasoup.find('dd',{'class':"category tags"})
+        if a != None:
+            genres = a.findAll('a',{'class':"tag"})
+            for genre in genres:
+                if genre != "Gen":
+                    self.story.addToList('genre',genre.string)
+		
+        a = metasoup.find('dd',{'class':"character tags"})
+        if a != None:
+            chars = a.findAll('a',{'class':"tag"})
+            for char in chars:
+                self.story.addToList('characters',char.string)
+        a = metasoup.find('dd',{'class':"relationship tags"})
+        if a != None:
+            chars = a.findAll('a',{'class':"tag"})
+            for char in chars:
+                self.story.addToList('characters',char.string)
+		
+
+        stats = metasoup.find('dl',{'class':'stats'})
+        dt = stats.findAll('dt')
+        dd = stats.findAll('dd')
+        for x in range(0,len(dt)):
+            label = dt[x].text
+            value = dd[x].text
+
+            if 'Words:' in label:
+                self.story.setMetadata('numWords', value)
+				
+            if 'Chapters:' in label:
+                if value.split('/')[0] == value.split('/')[1]:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+				
+            if 'Completed' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+		
+        try:
+            # Find Series name from series URL.
+            a = metasoup.find('dd',{'class':"series"})
+            b = a.find('a', href=re.compile(r"/series/\d+"))
+            series_name = b.string
+            series_url = 'http://'+self.host+'/fanfic/'+b['href']
+            series_index = int(a.text.split(' ')[1])
+            self.setSeries(series_name, series_index)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
+		
+        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
+        if headnotes != None:
+            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+            if headnotes != None:
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(headnotes)
+        
+        chapsumm = soup.find('div', {'id' : "summary"})
+        if chapsumm != None:
+            chapsumm = chapsumm.find('blockquote')
+            chapter.append("<b>Summary for the Chapter:</b>")
+            chapter.append(chapsumm)
+        chapnotes = soup.find('div', {'id' : "notes"})
+        if chapnotes != None:
+            chapnotes = chapnotes.find('blockquote')
+            if chapnotes != None:
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapnotes)
+		
+        text = soup.find('div', {'class' : "userstuff module"})
+        chtext = text.find('h3', {'class' : "landmark heading"})
+        if chtext:
+            chtext.extract()
+        chapter.append(text)
+		
+        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+        if chapfoot != None:
+            chapfoot = chapfoot.find('blockquote')
+            chapter.append("<b>Notes for the Chapter:</b>")
+            chapter.append(chapfoot)
+		
+        footnotes = soup.find('div', {'id' : "work_endnotes"})
+        if footnotes != None:
+            footnotes = footnotes.find('blockquote')
+            chapter.append("<b>Author's Note:</b>")
+            chapter.append(footnotes)
+			
+        if None == soup:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chapter)

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -24,7 +24,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 # By virtue of being recent and requiring both is_adult and user/pass,
 # adapter_fanficcastletvnet.py is the best choice for learning to
@@ -74,6 +74,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
         self._setURL('http://' + self.getSiteDomain() + '/fanfic/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
@@ -218,7 +219,8 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -272,6 +274,26 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
@@ -285,4 +307,4 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -24,7 +24,7 @@ import time
 from .. import BeautifulSoup as bs
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
@@ -70,10 +70,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         url = self.origurl
         logging.debug("URL: "+url)
-        
+
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
+            #print("\n===================\n%s\n===================\n"%data)
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -83,10 +84,33 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             
         if "Unable to locate story with id of " in data:
             raise exceptions.StoryDoesNotExist(url)
-            
-        if "Chapter not found. Please check to see you are not using an outdated url." in data:
+
+        # some times "Chapter not found...", sometimes "Chapter text not found..."
+        if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
-            
+
+        try:
+            # rather nasty way to check for a newer chapter.  ffnet has a
+            # tendency to send out update notices in email before all
+            # their servers are showing the update on the first chapter.
+            try:
+                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+            # get chapter part of url.
+            except:
+                chapcount = 1
+            chapter = url.split('/',)[5]
+            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                           self.story.getMetadata('storyId'),
+                                           chapcount+1)
+            print('=Trying newer chapter: %s' % tryurl)
+            newdata = self._fetchUrl(tryurl)
+            if "not found. Please check to see you are not using an outdated url." \
+                    not in newdata:
+                print('=======Found newer chapter: %s' % tryurl)
+                soup = bs.BeautifulSoup(newdata)
+        except:
+            pass
+        
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -118,7 +142,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 continue
             if 'var storyid' in script.string:
                 for line in script.string.split('\n'):
-                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;$",line)
+                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;\r?$",line)
                     if m == None : continue
                     var,value = m.groups()
                     # remove javascript escaping from values.
@@ -129,7 +153,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                     if 'title_t' in var:
                         self.story.setMetadata('title', value)
                     if 'summary' in var:
-                        self.story.setMetadata('description', value)
+                        self.setDescription(url,value)
+                        #self.story.setMetadata('description', value)
                     if 'datep' in var:
                         self.story.setMetadata('datePublished',makeDate(value, '%m-%d-%y'))
                     if 'dateu' in var:
@@ -179,7 +204,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         metatext = a.findNext(text=re.compile(r' - Reviews:'))
         if metatext == None: # indicates there's no Reviews, look for id: instead.
             metatext = a.findNext(text=re.compile(r' - id:'))
-        #print("========= metatext:\n%s"%metatext)
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
@@ -193,7 +217,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
         # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
         # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?:[ ]+with characters (?P<char1>.*?\.?)(?: & (?P<char2>.*?\.?))?\. )?",
+        # <meta name="description" content="Chapter 1 of a Harry Potter and Transformers  - Humor/Adventure crossover fanfiction  with characters: Harry P. & Ironhide. IT’s one thing to be tossed thru the Veil for something he didn’t do. It was quite another to wake in his animigus form in a world not his own. Harry just knew someone was laughing at him somewhere. Mech/Mech pairings inside..">
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
                      soup.find('meta',{'name':'description'})['content'])
         if m != None:
             genres=m.group('genres')
@@ -203,7 +228,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 for g in genres.split('/'):
                     self.story.addToList('genre',g)
 
-            if m.group('char1') != None:
+            if m.group('chars') != None:
+
                 # At this point we've proven that there's character(s)
                 # We can't reliably parse characters out of meta name="description".
                 # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
@@ -211,15 +237,18 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 # reviewstext can take form of:
                 # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
                 # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
+                # - English - Humor/Adventure -  Harry P. & Ironhide - Reviews:
                 mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
                               metatext)
                 chars = mc.group("chars")
                 for c in chars.split(' & '):
                     self.story.addToList('characters',c)
-        
+        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
+        if m.group('lang') != None:
+            self.story.setMetadata('language',m.group('lang'))
+                    
         return
 
-
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
@@ -242,7 +271,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             logging.debug('div id=storytext not found.  data:%s'%data)
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)
 
 def getClass():
     return FanFictionNetSiteAdapter

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -0,0 +1,222 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import datetime
+import logging
+import re
+import urllib2
+from .. import translit
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return FicBookNetAdapter
+
+
+class FicBookNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fbn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %m %Y" 
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ficbook.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/readfic/12345"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        url=self.url
+        logging.debug("URL: "+url)
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+				
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+		
+        table = soup.find('td',{'width':'50%'})
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
+        
+        # Find authorid and URL from... author url.
+        a = table.find('a')
+        self.story.setMetadata('authorId',a.text) # Author's name is unique
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.text)
+        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
+
+        # Find the chapters:
+        chapters = soup.find('div', {'class' : 'part_list'})
+        if chapters != None:
+            chapters=chapters.findAll('a', href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+"/\d+#part_content$"))
+            self.story.setMetadata('numChapters',len(chapters))
+            for x in range(0,len(chapters)):
+                chapter=chapters[x]
+                churl='http://'+self.host+chapter['href']
+                self.chapterUrls.append((stripHTML(chapter),churl))
+                if x == 0:
+                    pubdate = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+                if x == len(chapters)-1:
+                    update = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            self.story.setMetadata('numChapters',1)
+            pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
+            update=pubdate
+
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+
+        if not ',' in pubdate:
+            pubdate=datetime.date.today().strftime(self.dateformat)
+        if not ',' in update:
+            update=datetime.date.today().strftime(self.dateformat)
+        pubdate=pubdate.split(',')[0]
+        update=update.split(',')[0]
+		
+        fullmon = {"yanvarya":"01", "января":"01",
+           "fievralya":"02", "февраля":"02",
+           "marta":"03", "марта":"03",
+           "aprielya":"04", "апреля":"04",
+           "maya":"05", "мая":"05",
+           "iyunya":"06", "июня":"06",
+           "iyulya":"07", "июля":"07",
+           "avghusta":"08", "августа":"08",
+           "sentyabrya":"09", "сентября":"09",
+           "oktyabrya":"10", "октября":"10",
+           "noyabrya":"11", "ноября":"11",
+           "diekabrya":"12", "декабря":"12" }
+        for (name,num) in fullmon.items():
+            if name in pubdate:
+                pubdate = pubdate.replace(name,num)
+            if name in update:
+                update = update.replace(name,num)
+
+        self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
+        self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
+        self.story.setMetadata('language','Russian')
+		
+        pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
+        pr='http://'+self.host+pr['href']
+        pr = bs.BeautifulSoup(self._fetchUrl(pr))
+        pr=pr.findAll('div', {'class' : 'part_text'})
+        i=0
+        for part in pr:
+            i=i+len(stripHTML(part).split(' '))
+        self.story.setMetadata('numWords', str(i))
+		
+        i=0
+        fandoms = table.findAll('a', href=re.compile(r'/fanfiction/\w+'))
+        for fandom in fandoms:
+            self.story.addToList('category',fandom.string)
+            i=i+1
+        if i > 1:
+            self.story.addToList('genre', 'Кроссовер')
+		
+        meta=table.findAll('a', href=re.compile(r'/ratings/'))
+        i=0
+        for m in meta:
+            if i == 0:
+                self.story.setMetadata('rating', m.find('b').text)
+                i=1
+            elif i == 1:
+                if not "," in m.nextSibling:
+                    i=2
+                self.story.addToList('genre', m.find('b').text)
+            elif i == 2:
+                self.story.addToList('warnings', m.find('b').text)
+		
+
+        if table.find('span', {'style' : 'color: green'}):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In Progress')
+		
+
+        tags = table.findAll('b')
+        for tag in tags:
+            label = translit.translit(tag.text)
+            if 'Piersonazhi:' in label or 'Персонажи:' in label:
+                chars=tag.nextSibling.string.split(', ')
+                for char in chars:
+                    self.story.addToList('characters',char)
+                break
+				
+        summary=soup.find('span', {'class' : 'urlize'})
+        self.setDescription(url,summary.text)
+        #self.story.setMetadata('description', summary.text)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        chapter = soup.find('div', {'class' : 'public_beta'})
+        if chapter == None:
+            chapter = soup.find('div', {'class' : 'public_beta_disabled'})
+
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,chapter)

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
@@ -187,7 +187,8 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             
         for small in storydd.findAll('small'):
             small.extract() ## removes the <small> tags, leaving only the summary.
-        self.story.setMetadata('description',stripHTML(storydd))
+        self.setDescription(url,storydd)
+        #self.story.setMetadata('description',stripHTML(storydd))
         
         return
 
@@ -203,6 +204,10 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 	# our div with poor html inside the story text.
 	data = data.replace('<!-- headerend -->','<crazytagstringnobodywouldstumbleonaccidently id="storytext">').replace('<!-- footerstart -->','</crazytagstringnobodywouldstumbleonaccidently>')
         
+        # problems with some stories confusing Soup.  This is a nasty
+        # hack, but it works.
+        data = data[data.index("<crazytagstringnobodywouldstumbleonaccidently"):]
+
         soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         body = soup.findAll('body') ## some stories use a nested body and body
@@ -219,7 +224,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if not data or not text:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(text)
+        return self.utf8FromSoup(url,text)
 
 def getClass():
     return FictionAlleyOrgSiteAdapter

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -26,7 +26,7 @@ from .. import BeautifulSoup as bs
 from .. import exceptions as exceptions
 from ..htmlcleanup import stripHTML
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class FicwadComSiteAdapter(BaseSiteAdapter):
 
@@ -124,7 +124,8 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # description
         storydiv = soup.find("div",{"id":"story"})
-        self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
+        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p.string)
+        #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
 
         # most of the meta data is here:
         metap = storydiv.find("p",{"class":"meta"})
@@ -201,9 +202,6 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet tends to fail more if hit too fast.
-                        ## This is in additional to what ever the
-                        ## slow_down_sleep_time setting is.
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 
@@ -212,7 +210,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)
 
 def getClass():
     return FicwadComSiteAdapter

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -20,12 +20,13 @@ import logging
 import re
 import urllib2
 import cookielib as cl
+import datetime
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
     return FimFictionNetSiteAdapter
@@ -82,9 +83,23 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         
         if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
             raise exceptions.StoryDoesNotExist(self.url)
+
+        if "/images/missing_story.png" in data:
+            raise exceptions.StoryDoesNotExist(self.url)
         
         if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
+
+        if self.password:
+            params = {}
+            params['password'] = self.password
+            data = self._postUrl(self.url,params)
+
+        if "Enter the password the author set for this story to view it." in data:
+            if self.getConfig('fail_on_password'):
+                raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
+            else:
+                raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
         
         soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"})
 
@@ -101,13 +116,13 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         for chapter in soup.findAll("a", {"class":"chapter_link"}):
             chapterDates.append(chapter.span.extract().text.strip("()"))
             self.chapterUrls.append((chapter.text.strip(), "http://"+self.getSiteDomain() + chapter['href']))
-        
+
         self.story.setMetadata('numChapters',len(self.chapterUrls))
         
         for character in [character_icon['title'] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
             self.story.addToList("characters", character)
         for category in [category.text for category in soup.find("div", {"class":"categories"}).findAll("a")]:
-            self.story.addToList("category", category)
+            self.story.addToList("genre", category)
         self.story.addToList("category", "My Little Pony")
         
         
@@ -140,26 +155,45 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             description_soup.find('a', {"class":"more"}).extract()
         except:
             pass
-        self.story.setMetadata('description', description_soup.text)
-        
-        # Unfortunately, nowhere on the page is the year mentioned. Because we would much rather update the story needlessly
-        # than miss an update, we hardcode the year of creation and update to be 2011.
 
-        # Get the date of creation from the first chapter
-        datePublished_text = chapterDates[0]
-        day, month = datePublished_text.split()
-        day = re.sub(r"[^\d.]+", '', day)
-        datePublished = makeDate("2011"+month+day, "%Y%b%d")
-        self.story.setMetadata("datePublished", datePublished)
+        # fimfic is the first site with an explicit cover image.
+        story_img = soup.find('img',{'class':'story_image'})
+        if self.getConfig('include_images') and story_img:
+            self.story.addImgUrl(self,self.url,story_img['src'],self._fetchUrlRaw,cover=True)
+        self.setDescription(self.url,description_soup.text)
+        #self.story.setMetadata('description', description_soup.text)
+        
+        # Unfortunately, nowhere on the page is the year mentioned.
+        # Best effort to deal with this:
+        # Use this year, if that's a date in the future, subtract one year.
+        # Their earliest story is Jun, so they'll probably change the date
+        # around then.
+
+        now = datetime.datetime.now()
+        
         dateUpdated_soup = bs.BeautifulSoup(data).find("div", {"class":"calendar"})
         dateUpdated_soup.find('span').extract()
-        dateUpdated = makeDate("2011"+dateUpdated_soup.text, "%Y%b%d")
+        dateUpdated = makeDate("%s%s"%(now.year,dateUpdated_soup.text), "%Y%b%d")
+        if dateUpdated > now :
+            dateUpdated = dateUpdated.replace(year=now.year-1)
         self.story.setMetadata("dateUpdated", dateUpdated)
         
+        # Get the date of creation from the first chapter
+        if len(chapterDates) > 0:
+            datePublished_text = chapterDates[0]
+            day, month = datePublished_text.split()
+            day = re.sub(r"[^\d.]+", '', day)
+            datePublished = makeDate("%s%s%s"%(now.year,month,day), "%Y%b%d")
+            if datePublished > now :
+                datePublished = datePublished.replace(year=now.year-1)
+            self.story.setMetadata("datePublished", datePublished)
+        else:
+            self.story.setMetadata("datePublished", dateUpdated)
+            
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-        return utf8FromSoup(soup)
+        return self.utf8FromSoup(url,soup)
         

fanficdownloader/adapters/adapter_gayauthorsorg.py

@@ -0,0 +1,205 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import logging
+import re
+import urllib2
+from urllib import unquote
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+def getClass():
+    return GayAuthorsAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class GayAuthorsAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+       
+		
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        # unqoute, change '_' and ' ' to '-', downcase, and remove non-[a-z0-9-]
+        authid = unquote(self.parsedUrl.path.split('/',)[2])
+        authid = authid.lower().replace('_','-').replace(' ','-')
+        authid = re.sub(r"[^a-z0-9-]","",authid)
+        
+        self.story.setMetadata('authorId',authid)
+        logging.debug("authorId: (%s)"%self.story.getMetadata('authorId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('authorId') + '/' + self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ga')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.gayauthors.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story/author/storytitle"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story/")+r".*?/\w+.*?$"
+
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        msoup = soup.find('div', {'class' : 'story'})
+        if msoup == None:
+            msoup = soup.find('div', {'class' : 'story ispinned'})
+        csoup = soup.find('div', {'id' : 'story_chapters'})
+        
+        ## Title
+        a = msoup.find('span', {'class' : 'title'})
+        title=a.find('span', {'itemprop' : 'name'})
+        self.story.setMetadata('title',title.text)
+        try:
+            # Find Series name from series URL.
+            series = a.find('span',{'class':"description"})
+            series_name = series.find('a')
+            series_name.extract()
+            series_index = int(series.text.split(' ')[1])
+            self.setSeries(series_name.text, series_index)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+        
+        # Find authorid and URL from... author url.
+        a = msoup.find('a', href=re.compile(r'/author/'+self.story.getMetadata('authorId')))
+        self.story.setMetadata('authorUrl',a['href'])
+        self.story.setMetadata('author',a.text)
+		
+
+        # Find the chapters:
+        spans=csoup.findAll('span', {'class' : 'desc chapter-info'})
+        for span in spans:
+            span.extract()
+        for chapter in csoup.findAll('a'):
+            # just in case there's tags, like <i> in chapter titles.
+            a=chapter['href'].split(self.story.getMetadata('author'))
+            a=a[0]+self.story.getMetadata('authorId')+a[1]
+            self.chapterUrls.append((stripHTML(chapter),a))
+
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+		
+        cats = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_0__category\w+$'))
+        for cat in cats:
+            self.story.addToList('category',cat.text)
+		
+        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_1__genre\w+$'))
+        for genre in genres:
+            self.story.addToList('genre',genre.text)
+        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_2__tag\w+$'))
+        for genre in genres:
+            self.story.addToList('genre',genre.text)
+			
+        status = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_3__status\w+$'))
+        self.story.setMetadata('status',status.text)
+		
+        rating = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_4__rating\w+$'))
+        self.story.setMetadata('rating',rating.text)
+		
+        summary = msoup.find('span', {'itemprop' : 'description'})
+        self.setDescription(self.url,summary.text)
+        #self.story.setMetadata('description',summary.text)
+	
+
+        stats = msoup.find('dl',{'class':'info'})
+        dt = stats.findAll('dt')
+        dd = stats.findAll('dd')
+        for x in range(0,len(dt)):
+            label = dt[x].text
+            value = dd[x].text
+
+            if 'Words:' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Published:' in label:
+                date=stripHTML(value.split(' - ')[0])
+                if ',' in date:
+                    date=datetime.date.today().strftime(self.dateformat)
+                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+				
+            if 'Updated:' in label:
+                date=stripHTML(value.split(' - ')[0])
+                if ',' in date:
+                    date=datetime.date.today().strftime(self.dateformat)
+                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        data = data[data.index("<div id='chapter-content'>"):]
+        soup = bs.BeautifulSoup(data) # this one's happier with Soup, not StoneSoup for some reason.
+        
+        div = soup.find('div', {'id' : 'chapter-content'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
@@ -125,7 +125,8 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Finding the metadata is a bit of a pain.  Desc is the only thing this color.
         desctable= soup.find('table',{'bgcolor':'#f0e8e8'})
-        self.story.setMetadata('description',stripHTML(desctable))
+        self.setDescription(url,desctable)
+        #self.story.setMetadata('description',stripHTML(desctable))
 
         ## Finding the metadata is a bit of a pain.  Most of the meta
         ## data is in a center.table without a bgcolor.
@@ -193,7 +194,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)
 
 def getClass():
     return HarryPotterFanFictionComSiteAdapter

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return HPFandomNetAdapterAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/eff/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hpfdm') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.hpfandom.net' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/eff/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        ## Going to get the rest from the author page.
+        authdata = self._fetchUrl(self.story.getMetadata('authorUrl'))
+        # fix a typo in the site HTML so I can find the Characters list.
+        authdata = authdata.replace('<td width=10%">','<td width="10%">')
+
+        # hpfandom.net only seems to indicate adult-only by javascript on the story/chapter links.
+        if "javascript:if (confirm('Slash/het fiction which incorporates sexual situations to a somewhat graphic degree and some violence. ')) location = 'viewstory.php?sid=%s'"%self.story.getMetadata('storyId') in authdata \
+                and not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        authsoup = bs.BeautifulSoup(authdata)
+
+        reviewsa = authsoup.find('a', href="reviews.php?sid="+self.story.getMetadata('storyId')+"&a=")
+        # <table><tr><td><p><b><a ...>
+        metablock = reviewsa.findParent("table")
+        #print("metablock:%s"%metablock)
+        
+        ## Title
+        titlea = metablock.find('a', href=re.compile("viewstory.php"))
+        #print("titlea:%s"%titlea)
+        if titlea == None:
+            raise exceptions.FailedToDownload("Story URL (%s) not found on author's page, can't use chapter URLs"%url)
+        self.story.setMetadata('title',stripHTML(titlea))
+        
+        # Find the chapters: !!! hpfandom.net differs from every other
+        # eFiction site--the sid on viewstory for chapters is
+        # *different* for each chapter
+        for chapter in soup.findAll('a', {'href':re.compile(r"viewstory.php\?sid=\d+&i=\d+")}):
+            m = re.match(r'.*?(viewstory.php\?sid=\d+&i=\d+).*?',chapter['href'])
+            # just in case there's tags, like <i> in chapter titles.
+            #print("====chapter===%s"%m.group(1))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eff/'+m.group(1)))
+
+        if len(self.chapterUrls) == 0:
+            self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        summary = metablock.find("td",{"class":"summary"})
+        summary.name='span'
+        self.setDescription(url,summary)
+
+        # words & completed in first row of metablock.
+        firstrow = stripHTML(metablock.find('tr'))
+        # A Mother's Love xx Going Grey 1 (G+) by Kiristeen | Reviews - 18 | Words: 27468 | Completed: Yes
+        m = re.match(r".*?\((?P<rating>[^)]+)\).*?Words: (?P<words>\d+).*?Completed: (?P<status>Yes|No)",firstrow)
+        if m != None:
+            if m.group('rating') != None:
+                self.story.setMetadata('rating', m.group('rating'))
+                
+            if m.group('words') != None:
+                self.story.setMetadata('numWords', m.group('words'))
+
+            if m.group('status') != None:
+                if 'Yes' in m.group('status'):
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                
+
+        # <tr><td width="10%" valign="top">Chapters:</td><td width="40%" valign="top">4</td>
+        # <td width="10%" valign="top">Published:</td><td width="40%" valign="top">2010.09.29</td></tr>
+        # <tr><td width="10%" valign="top">Completed:</td><td width="40%" valign="top">Yes</td><td width="10%" valign="top">Updated:</td><td width="40%" valign="top">2010.10.03</td></tr>
+        labels = metablock.findAll('td',{'width':'10%'})
+        for td in labels:
+            label = td.string
+            value = td.nextSibling.string
+            #print("\nlabel:%s\nvalue:%s\n"%(label,value))
+
+            if 'Category' in label:
+                cats = td.parent.findAll('a',href=re.compile(r'categories.php'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.split(','):
+                    self.story.addToList('characters',char.strip())
+
+            if 'Genre' in label:
+                for genre in value.split(','):
+                    self.story.addToList('genre',genre.strip())
+
+            if 'Warnings' in label:
+                for warning in value.split(','):
+                    if warning.strip() != 'none':
+                        self.story.addToList('warnings',warning.strip())
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        # There's no good wrapper around the chapter text. :-/
+        # There are, however, tables with width=100% just above and below the real text.
+        data = re.sub(r'<table width="100%">.*?</table>','<div name="storybody">',
+                      data,count=1,flags=re.DOTALL)
+        
+        data = re.sub(r'<table width="100%">.*?</table>','</div>',
+                      data,count=1,flags=re.DOTALL)
+        
+        soup = bs.BeautifulStoneSoup(data,selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        div = soup.find("div",{'name':'storybody'})
+        #print("\n\ndiv:%s\n\n"%div)
+        
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
@@ -174,7 +174,8 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         # Summary: ....
         m = re.match(r".*?Summary: (.*)$",metastr) 
         if m:
-            self.story.setMetadata('description', m.group(1))
+            self.setDescription(url, m.group(1))
+            #self.story.setMetadata('description', m.group(1))
 
         # completed
         m = re.match(r".*?Status: Completed.*?",metastr)
@@ -210,7 +211,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 del div['style']
                 del div['align']
             anchor.name='div'
-            return utf8FromSoup(anchor)
+            return self.utf8FromSoup(url,anchor)
         
         else:
             logging.debug('Using kludgey text find for older mediaminer story.')
@@ -226,7 +227,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                     soup.findAll('table',{'class':'tbbrdr'}):
                 tag.extract() # remove tag from soup.
                 
-            return utf8FromSoup(soup)
+            return self.utf8FromSoup(url,soup)
         
 
 def getClass():

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return MuggleNetComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MuggleNetComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mgln') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfiction.mugglenet.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "This story may contain some sexuality, violence and or profanity not suitable for younger readers." in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+
+        # Dunno if this site uses this or not.
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -0,0 +1,274 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+import cookielib as cl
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return PortkeyOrgAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','prtky') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfiction.portkey.org' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story/1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story/")+r"\d+(/\d+)?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        # portkey screws around with using a different URL to set the
+        # cookie and it's a pain.  So... cheat!
+        if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
+            cookie = cl.Cookie(version=0, name='verify17', value='1',
+                               port=None, port_specified=False,
+                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
+                               path='/', path_specified=True,
+                               secure=False,
+                               expires=time.time()+10000,
+                               discard=False,
+                               comment=None,
+                               comment_url=None,
+                               rest={'HttpOnly': None},
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if "You must be over 18 years of age to view it" in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/profile/\d+"))
+        #print("======a:%s"%a)
+        self.story.setMetadata('authorId',a['href'].split('/')[-1])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        ## Going to get the rest from the author page.
+        authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        ## Title
+        titlea = authsoup.find('a', href=re.compile(r'/story/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(titlea))
+        metablock = titlea.parent
+        
+        # Find the chapters:
+        for chapter in soup.find('select',{'name':'select5'}).findAll('option', {'value':re.compile(r'/story/'+self.story.getMetadata('storyId')+"/\d+$")}):
+            # just in case there's tags, like <i> in chapter titles.
+            chtitle = stripHTML(chapter)
+            if not chtitle:
+                chtitle = "(Untitled Chapter)"
+            self.chapterUrls.append((chtitle,'http://'+self.host+chapter['value']))
+            
+        if len(self.chapterUrls) == 0:
+            self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+        # <SPAN class="dark-small-bold">Contents:</SPAN> <SPAN class="small-grey">NC17 </SPAN> 
+        # <SPAN class="dark-small-bold">Published: </SPAN><SPAN class="small-grey">12/11/07</SPAN>
+        # <SPAN class="dark-small-bold"><BR>
+        # Description:</SPAN> <SPAN class="small-black">A special book helps Harry tap into the power the Dark Lord knows not.  Of course it’s a book on sex magic and rituals… but Harry’s not complaining.  Spurned on by the ghost of a pervert founder, Harry leads his friends in the hunt for Voldemort’s Horcruxes.
+        # EROTIC COMEDY!  Loads of crude humor and sexual situations!
+        # </SPAN>
+        labels = metablock.findAll('span',{'class':'dark-small-bold'})
+        for labelspan in labels:
+            value = labelspan.findNext('span').string
+            label = stripHTML(labelspan)
+#            print("\nlabel:%s\nlabel:%s\nvalue:%s\n"%(labelspan,label,value))
+
+            if 'Description' in label:
+                self.setDescription(url,value)
+
+            if 'Contents' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Words' in label:
+                self.story.setMetadata('numWords', value)
+
+            # if 'Categories' in label:
+            #     cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+            #     catstext = [cat.string for cat in cats]
+            #     for cat in catstext:
+            #         self.story.addToList('category',cat.string)
+
+            # if 'Characters' in label:
+            #     chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+            #     charstext = [char.string for char in chars]
+            #     for char in charstext:
+            #         self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                # genre is typo'ed on the site--it falls between the
+                # dark-small-bold label and dark-small-bold content
+                # spans.
+                svalue = ""
+                value = labelspan.nextSibling
+                while not defaultGetattr(value,'class') == 'dark-small-bold':
+                    svalue += str(value)
+                    value = value.nextSibling
+                    
+                for genre in svalue.split("/"):
+                    genre = genre.strip()
+                    if genre != 'None':
+                        self.story.addToList('genre',genre)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            # if 'Warnings' in label:
+            #     warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+            #     warningstext = [warning.string for warning in warnings]
+            #     self.warning = ', '.join(warningstext)
+            #     for warning in warningstext:
+            #         self.story.addToList('warnings',warning.string)
+
+            if 'Status' in label:
+                if 'Completed' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        # try:
+        #     # Find Series name from series URL.
+        #     a = metablock.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+        #     series_name = a.string
+        #     series_url = 'http://'+self.host+'/'+a['href']
+
+        #     # use BeautifulSoup HTML parser to make everything easier to find.
+        #     seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+        #     storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+        #     i=1
+        #     for a in storyas:
+        #         if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+        #             self.setSeries(series_name, i)
+        #             break
+        #         i+=1            
+        # except:
+        #     # I find it hard to care if the series parsing fails
+        #     pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        #print("soup:%s"%soup)
+        td = soup.find('td', {'class' : 'story'})
+
+        centers = td.findAll('center')
+        # first two and last two center tags are some script, 'report
+        # story', 'report story' and an ad.
+        centers[0].extract()
+        centers[1].extract()
+        centers[-1].extract()
+        centers[-2].extract()
+
+        if None == td:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,td)

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
 
@@ -61,14 +61,6 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"
 
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-          return True
-        else:
-          return False
-
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
@@ -131,7 +123,8 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 while not defaultGetattr(value,'class') == 'listbox':
                     svalue += str(value)
                     value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -176,7 +169,27 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
 
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
@@ -189,7 +202,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)
 
 def getClass():
     return PotionsAndSnitchesNetSiteAdapter

fanficdownloader/adapters/adapter_siyecouk.py

@@ -24,7 +24,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 # This function is called by the downloader in all adapter_*.py files
 # in this dir to register the adapter class.  So it needs to be
@@ -45,9 +45,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        # self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        # self.password = ""
-        # self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
@@ -81,55 +78,11 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?"+re.escape("siye.co.uk/siye/viewstory.php?sid=")+r"\d+$"
-
-    # ## Login seems to be reasonably standard across eFiction sites.
-    # def needToLoginCheck(self, data):
-    #     if 'Registered Users Only' in data \
-    #             or 'There is no such account on our website' in data \
-    #             or "That password doesn't match the one in our database" in data:
-    #         return True
-    #     else:
-    #         return False
-        
-    # def performLogin(self, url):
-    #     params = {}
-
-    #     if self.password:
-    #         params['penname'] = self.username
-    #         params['password'] = self.password
-    #     else:
-    #         params['penname'] = self.getConfig("username")
-    #         params['password'] = self.getConfig("password")
-    #     params['cookiecheck'] = '1'
-    #     params['submit'] = 'Submit'
-    
-    #     loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-    #     logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-    #                                                           params['penname']))
-    
-    #     d = self._fetchUrl(loginUrl, params)
-    
-    #     if "Member Account" not in d : #Member Account
-    #         logging.info("Failed to login to URL %s as %s" % (loginUrl,
-    #                                                           params['penname']))
-    #         raise exceptions.FailedToLogin(url,params['penname'])
-    #         return False
-    #     else:
-    #         return True
+        return re.escape("http://")+r"(www\.)?siye\.co\.uk/(siye/)?"+re.escape("viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
-        # if self.is_adult or self.getConfig("is_adult"):
-        #     # Weirdly, different sites use different warning numbers.
-        #     # If the title search below fails, there's a good chance
-        #     # you need a different number.  print data at that point
-        #     # and see what the 'click here to continue' url says.
-        #     addurl = "&ageconsent=ok&warning=4" # XXX
-        # else:
-        #     addurl=""
-
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         # Except it doesn't this time. :-/
@@ -144,20 +97,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             else:
                 raise e
 
-        # if self.needToLoginCheck(data):
-        #     # need to log in for this one.
-        #     self.performLogin(url)
-        #     data = self._fetchUrl(url)
-
-        # # The actual text that is used to announce you need to be an
-        # # adult varies from site to site.  Again, print data before
-        # # the title search to troubleshoot.
-        # if "Age Consent Required" in data: # XXX 
-        #     raise exceptions.AdultCheckRequired(self.url)
-            
-        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data
@@ -227,16 +166,15 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
 
             if part.startswith("Summary:"):
                 part = part[part.find(':')+1:]
-                self.story.setMetadata('description',part)
-
-        
-
+                self.setDescription(url,part)
+                #self.story.setMetadata('description',part)
+                
         # want to get the next tr of the table.
         #print("%s"%titlea.parent.parent.findNextSibling('tr'))
             
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
-        moremeta = stripHTML(titlea.parent.parent.findNextSibling('tr'))
+        moremeta = stripHTML(titlea.parent.parent.parent.find('div',{'class':'desc'}))
         for part in moremeta.replace(' - ','\n').split('\n'):
             #print("part:%s"%part)
             try:
@@ -259,7 +197,25 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             if name == 'Words':
                 self.story.setMetadata('numWords', value)
 
-        
+        try:
+            # Find Series name from series URL.
+            a = titlea.findPrevious('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass        
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):
@@ -279,4 +235,4 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         if None == story:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(story)
+        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
 
@@ -164,7 +164,8 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -205,6 +206,26 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
 
     def getChapterText(self, url):
 
@@ -218,7 +239,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)
 
 def getClass():
     return TenhawkPresentsComSiteAdapter

fanficdownloader/adapters/adapter_test1.py

@@ -22,7 +22,7 @@ import logging
 from .. import BeautifulSoup as bs
 from .. import exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class TestSiteAdapter(BaseSiteAdapter):
 
@@ -78,13 +78,30 @@ class TestSiteAdapter(BaseSiteAdapter):
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
         self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
-        self.story.setMetadata('dateCreated',datetime.datetime.now())
         if self.story.getMetadata('storyId') == '669':
             self.story.setMetadata('dateUpdated',datetime.datetime.now())
         else:
             self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
-        self.story.setMetadata('status','In-Completed')
+
+        idnum = int(self.story.getMetadata('storyId'))
+        if idnum % 2 == 1:
+            self.story.setMetadata('status','In-Progress')
+        else:
+            self.story.setMetadata('status','Completed')
+
+        langs = {
+            0:"English",
+            1:"Russian",
+            2:"French",
+            3:"German",
+            }
+        if idnum < 10:
+            self.story.setMetadata('language',langs[idnum%len(langs)])
+        # greater than 10, no language.
+            
+        self.setSeries('The Great Test',idnum)
+        
         self.story.setMetadata('rating','Tweenie')
         
         self.story.setMetadata('authorId','98765')
@@ -96,7 +113,8 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('category','Harry Potter')
         self.story.addToList('category','Furbie')
         self.story.addToList('category','Crossover')
-        
+        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','SF')
         self.story.addToList('genre','Noir')
@@ -159,20 +177,22 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         else:
             text=u'''
 <div>
-<h3>Chapter</h3>
+<h3>Chapter title from site</h3>
 <p><center>Centered text</center></p>
 <p>Lorem '''+self.crazystring+''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 br breaks<br><br>
+
+<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim &amp; Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a><br/>
 br breaks<br><br>
 <hr>
 horizontal rules
-<hr>
+<hr size=1 noshade>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 </div>
 '''
         soup = bs.BeautifulStoneSoup(text,selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        return utf8FromSoup(soup)
+        return self.utf8FromSoup(url,soup)
 
 def getClass():
     return TestSiteAdapter

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -0,0 +1,293 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return TheQuidditchPitchOrgAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the  part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tqdpch') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thequidditchpitch.org' # XXX
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.thequidditchpitch.org','thequidditchpitch.org']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only - Not suitable for readers under the age of legal consent in their country.' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if ("Not suitable for readers under the age of legal consent in their country." in data \
+                or "Not suitable for readers under 16 yrs. \r\nStories may contain violence, slight nudity, and/or sexual situations." in data ) \
+                and not (self.is_adult or self.getConfig("is_adult")): # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        # span?  Really?  span?  Yeah... I don't think so.
+        div = soup.find('span', {'style' : 'font-size: 100%;'})
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
 
@@ -166,7 +166,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -207,7 +207,26 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/library/'+a['href']
 
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
@@ -226,7 +245,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)
 
 def getClass():
     return TheWritersCoffeeShopComSiteAdapter

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
 
@@ -127,6 +127,8 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             else:
                 raise e
             
+        descurl = url
+            
         if "<h2>Story Not Found</h2>" in data:
             raise exceptions.StoryDoesNotExist(url)
         
@@ -154,12 +156,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             # going to pull part of the meta data from author list page.
             logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl'))
             authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
+            descurl=self.story.getMetadata('authorUrl')
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
                 nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
                 logging.debug("**AUTHOR** nextpage URL: "+nextpage)
                 authordata = self._fetchUrl(nextpage)
+                descurl=nextpage
                 authorsoup = bs.BeautifulSoup(authordata)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -168,20 +172,22 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 raise e
 
         storydiv = authorsoup.find('div', {'id':'st'+self.story.getMetadata('storyId'), 'class':re.compile(r"storylistitem")})
-        self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
+        self.setDescription(descurl,storydiv.find('div',{'class':'storydesc'}))
+        #self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
         self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))
 
         verticaltable = soup.find('table', {'class':'verticaltable'})
 
         BtVS = True
+        BtVSNonX = False
         for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
             if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
                 self.story.addToList('category',cat.string)
             else:
                 if 'Non-BtVS' in cat.string:
                     BtVS = False
-        if BtVS:
-            self.story.addToList('category','Buffy: The Vampire Slayer')
+                if 'BtVS/AtS Non-Crossover' == cat.string:
+                    BtVSNonX = True
 
         verticaltabletds = verticaltable.findAll('td')
         self.story.setMetadata('rating', verticaltabletds[2].string)
@@ -199,6 +205,13 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         for icon in storydiv.find('span',{'class':'storyicons'}).findAll('img'):
             if( icon['title'] not in ['Non-Crossover'] ) :
                 self.story.addToList('genre',icon['title'])
+            else:
+                if not BtVSNonX:
+                    BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover
+
+        print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
+        if BtVS:
+            self.story.addToList('category','Buffy: The Vampire Slayer')
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapnav' } )
@@ -214,6 +227,12 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 self.chapterUrls.append((stripHTML(o),url))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        pseries = soup.find('p', {'style':'margin-top:0px'})
+        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
+                     pseries.text)
+        if m:
+            self.setSeries(m.group('series'),m.group('num'))
         
         return
 
@@ -232,7 +251,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             div.find('h3').extract()
         except:
             pass
-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)
 
 def getClass():
     return TwistingTheHellmouthSiteAdapter

fanficdownloader/adapters/adapter_twilightednet.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class TwilightedNetSiteAdapter(BaseSiteAdapter):
 
@@ -162,7 +162,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -204,6 +204,25 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+            
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+        
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
 
     def getChapterText(self, url):
 
@@ -224,7 +243,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)
 
 def getClass():
     return TwilightedNetSiteAdapter

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -25,7 +25,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
@@ -169,7 +169,8 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -217,6 +218,25 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                 value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
 
     def getChapterText(self, url):
 
@@ -236,7 +256,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)
 
 def getClass():
     return TwiwriteNetSiteAdapter

fanficdownloader/adapters/adapter_whoficcom.py

@@ -23,7 +23,7 @@ import urllib2
 from .. import BeautifulSoup as bs
 from .. import exceptions as exceptions
 
-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
 
 class WhoficComSiteAdapter(BaseSiteAdapter):
 
@@ -120,9 +120,10 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         # link instead to find the appropriate metadata.
         a = soup.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
         metadata = a.findParent('td')
-        metadatachunks = utf8FromSoup(metadata).split('<br />')
+        metadatachunks = self.utf8FromSoup(None,metadata).split('<br />')
         # process metadata for this story.
-        self.story.setMetadata('description', metadatachunks[1])
+        self.setDescription(url,metadatachunks[1])
+        #self.story.setMetadata('description', metadatachunks[1])
 
         # First line of the stuff with ' - ' separators
         moremeta = metadatachunks[2]
@@ -189,6 +190,26 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             if name == 'Word Count':
                 self.story.setMetadata('numWords', value)
 
+        try:
+            # Find Series name from series URL.
+            a = metadata.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
@@ -204,7 +225,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)
 
 def getClass():
     return WhoficComSiteAdapter

fanficdownloader/adapters/base_adapter.py

@@ -22,6 +22,10 @@ import logging
 import urllib
 import urllib2 as u2
 import urlparse as up
+from functools import partial
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
 
 try:
     from google.appengine.api import apiproxy_stub_map
@@ -40,6 +44,7 @@ except:
     #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
 
 from ..story import Story
+from ..gziphttp import GZipProcessor
 from ..configurable import Configurable
 from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
 from ..exceptions import InvalidStoryURL
@@ -65,14 +70,15 @@ class BaseSiteAdapter(Configurable):
     def __init__(self, config, url):
         self.config = config
         Configurable.__init__(self, config)
-        self.addConfigSection(self.getSiteDomain())
-        self.addConfigSection("overrides")
+        self.setSectionOrder(self.getSiteDomain())
+        # self.addConfigSection(self.getSiteDomain())
+        # self.addConfigSection("overrides")
         
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
         
-        self.opener = u2.build_opener(u2.HTTPCookieProcessor())
+        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         self.storyDone = False
         self.metadataDone = False
         self.story = Story()
@@ -81,6 +87,8 @@ class BaseSiteAdapter(Configurable):
         self.chapterUrls = [] # tuples of (chapter title,chapter url)
         self.chapterFirst = None
         self.chapterLast = None
+        self.oldchapters = None
+        self.oldimgs = None
         ## order of preference for decoding.
         self.decode = ["utf8",
                        "Windows-1252"] # 1252 is a superset of
@@ -149,6 +157,12 @@ class BaseSiteAdapter(Configurable):
                          headers=headers)
         return self._decode(self.opener.open(req).read())
 
+    def _fetchUrlRaw(self, url, parameters=None):
+        if parameters != None:
+            return self.opener.open(url,urllib.urlencode(parameters)).read()
+        else:
+            return self.opener.open(url).read()
+    
     # parameters is a dict()
     def _fetchUrl(self, url, parameters=None):
         if self.getConfig('slow_down_sleep_time'):
@@ -158,10 +172,7 @@ class BaseSiteAdapter(Configurable):
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
-                if parameters:
-                    return self._decode(self.opener.open(url,urllib.urlencode(parameters)).read())
-                else:
-                    return self._decode(self.opener.open(url).read())
+                return self._decode(self._fetchUrlRaw(url,parameters))
             except Exception, e:
                 excpt=e
                 logging.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
@@ -181,15 +192,32 @@ class BaseSiteAdapter(Configurable):
     def getStory(self):
         if not self.storyDone:
             self.getStoryMetadataOnly()
+
             for index, (title,url) in enumerate(self.chapterUrls):
                 if (self.chapterFirst!=None and index < self.chapterFirst) or \
                         (self.chapterLast!=None and index > self.chapterLast):
                     self.story.addChapter(removeEntities(title),
                                           None)
                 else:
+                    if self.oldchapters and index < len(self.oldchapters):
+                        data = self.utf8FromSoup(None,
+                                                 self.oldchapters[index],
+                                                 partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
+                    else:
+                        data = self.getChapterText(url)
                     self.story.addChapter(removeEntities(title),
-                                          removeEntities(self.getChapterText(url)))
+                                          removeEntities(data))
             self.storyDone = True
+            
+            # include image, but no cover from story, add default_cover_image cover.
+            if self.getConfig('include_images') and \
+                    not self.story.cover and \
+                    self.getConfig('default_cover_image'):
+                self.story.addImgUrl(self,
+                                     None,
+                                     self.getConfig('default_cover_image'),
+                                     self._fetchUrlRaw,
+                                     cover=True)
         return self.story
 
     def getStoryMetadataOnly(self):
@@ -229,17 +257,98 @@ class BaseSiteAdapter(Configurable):
         "Needs to be overriden in each adapter class."
         pass
 
+    # Just for series, in case we choose to change how it's stored or represented later.
+    def setSeries(self,name,num):
+        if self.getConfig('collect_series'):
+            self.story.setMetadata('series','%s [%s]'%(name, num))
+
+    def setDescription(self,url,svalue):
+        #print("\n\nsvalue:\n%s\n"%svalue)
+        if self.getConfig('keep_summary_html'):
+            if isinstance(svalue,str) or isinstance(svalue,unicode):
+                svalue = bs.BeautifulSoup(svalue)
+            self.story.setMetadata('description',self.utf8FromSoup(url,svalue))
+        else:
+            self.story.setMetadata('description',stripHTML(svalue))
+        #print("\n\ndescription:\n"+self.story.getMetadata('description')+"\n\n")
+
+    # This gives us a unicode object, not just a string containing bytes.
+    # (I gave soup a unicode string, you'd think it could give it back...)
+    # Now also does a bunch of other common processing for us.
+    def utf8FromSoup(self,url,soup,fetch=None):
+        if not fetch:
+            fetch=self._fetchUrlRaw
+
+        acceptable_attributes = ['href','name']
+        #print("include_images:"+self.getConfig('include_images'))
+        if self.getConfig('include_images'):
+            acceptable_attributes.extend(('src','alt','longdesc'))
+            for img in soup.findAll('img'):
+                # some pre-existing epubs have img tags that had src stripped off.
+                if img.has_key('src'):
+                    img['longdesc']=img['src']
+                    img['src']=self.story.addImgUrl(self,url,img['src'],fetch)
+
+        for attr in soup._getAttrMap().keys():
+            if attr not in acceptable_attributes:
+                del soup[attr] ## strip all tag attributes except href and name
+                
+        for t in soup.findAll(recursive=True):
+            for attr in t._getAttrMap().keys():
+                if attr not in acceptable_attributes:
+                    del t[attr] ## strip all tag attributes except href and name
+
+            # these are not acceptable strict XHTML.  But we do already have 
+	    # CSS classes of the same names defined
+            if t.name in ('u'):
+                t['class']=t.name
+                t.name='span'
+            if t.name in ('center'):
+                t['class']=t.name
+                t.name='div'
+	    # removes paired, but empty tags.
+            if t.string != None and len(t.string.strip()) == 0 :
+                t.extract()
+                
+        retval = soup.__str__('utf8').decode('utf-8')
+
+        if self.getConfig('replace_hr'):
+            # replacing a self-closing tag with a container tag in the
+            # soup is more difficult than it first appears.  So cheat.
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+
+        if self.getConfig('nook_img_fix'):
+            # if the <img> tag doesn't have a div or a p around it,
+            # nook gets confused and displays it on every page after
+            # that under the text for the rest of the chapter.
+            retval = re.sub(r"(?!<(div|p)>)\s*(?P<imgtag><img[^>]+>)\s*(?!</(div|p)>)",
+                            "<div>\g<imgtag></div>",retval)
+            
+        # Don't want body tags in chapter html--writers add them.
+        # This is primarily for epub updates.
+        return re.sub(r"</?body>\r?\n?","",retval)
+
 fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
            "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
            "November":"11", "December":"12" }
 
+def cachedfetch(realfetch,cache,url):
+    if url in cache:
+        print("cache hit")
+        return cache[url]
+    else:
+        return realfetch(url)
+    
+
 def makeDate(string,format):
     # Surprise!  Abstracting this turned out to be more useful than
     # just saving bytes.
 
     # fudge english month names for people who's locale is set to
     # non-english.  All our current sites date in english, even if
-    # there's non-english content.
+    # there's non-english content. -- ficbook.net now makes that a
+    # lie.  It has to do something even more complicated to get
+    # Russian month names correct everywhere.
     do_abbrev = "%b" in format
         
     if "%B" in format or do_abbrev:
@@ -253,24 +362,3 @@ def makeDate(string,format):
             
     return datetime.datetime.strptime(string,format)
 
-acceptable_attributes = ['href','name']
-
-# this gives us a unicode object, not just a string containing bytes.
-# (I gave soup a unicode string, you'd think it could give it back...)
-def utf8FromSoup(soup):
-    for t in soup.findAll(recursive=True):
-        for attr in t._getAttrMap().keys():
-            if attr not in acceptable_attributes:
-                del t[attr] ## strip all tag attributes except href and name
-        # these are not acceptable strict XHTML.  But we do already have 
-	# CSS classes of the same names defined in constants.py
-	if t.name in ('u'):
-            t['class']=t.name
-            t.name='span'
-        if t.name in ('center'):
-            t['class']=t.name
-            t.name='div'
-	# removes paired, but empty tags.
-        if t.string != None and len(t.string.strip()) == 0 :
-            t.extract()
-    return soup.__str__('utf8').decode('utf-8')

fanficdownloader/configurable.py

@@ -21,16 +21,21 @@ import ConfigParser
 # inherit from Configurable.  The config file(s) uses ini format:
 # [sections] with key:value settings.
 #
-# There's a [defaults] section which is overriden by the writer's
-# section [epub], which is overriden by the adapter's section for each
-# site.
+# writer does [defaults], [www.whofic.com], [epub], [www.whofic.com:epub], [overrides]
+#
+# Until a write is created, the adapter only has [defaults], [www.whofic.com], [overrides]
 #
 # [defaults]
 # titlepage_entries: category,genre, status
-# [epub]
-# titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
 # [www.whofic.com]
 # titlepage_entries: category,genre, status,dateUpdated,rating
+# [epub]
+# titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+# [www.whofic.com:epub]
+# titlepage_entries: category,genre, status,datePublished
+# [overrides]
+# titlepage_entries: category
+
 
 class Configurable(object):
 
@@ -38,6 +43,14 @@ class Configurable(object):
         self.config = config
         self.sectionslist = ['defaults']
 
+    def setSectionOrder(self,site,fileform=None):
+        self.sectionslist = ['defaults']
+        self.addConfigSection(site)
+        if fileform:
+            self.addConfigSection(fileform)
+            self.addConfigSection(site+":"+fileform)
+        self.addConfigSection("overrides")
+
     def addConfigSection(self,section):
         self.sectionslist.insert(0,section)
 

fanficdownloader/epubutils.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import re, os, traceback
+from zipfile import ZipFile
+from xml.dom.minidom import parseString
+
+from . import BeautifulSoup as bs
+
+def get_dcsource(inputio):
+    return get_update_data(inputio,getfilecount=False,getsoups=False)[0]
+
+def get_dcsource_chaptercount(inputio):
+    return get_update_data(inputio,getfilecount=True,getsoups=False)[:2] # (source,filecount)
+
+def get_update_data(inputio,
+                    getfilecount=True,
+                    getsoups=True):
+    epub = ZipFile(inputio, 'r')
+
+    ## Find the .opf file.
+    container = epub.read("META-INF/container.xml")
+    containerdom = parseString(container)
+    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
+    rootfilename = rootfilenodelist[0].getAttribute("full-path")
+
+    contentdom = parseString(epub.read(rootfilename))
+    firstmetadom = contentdom.getElementsByTagName("metadata")[0]
+    try:
+        source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8")
+    except:
+        source=None
+
+    ## Save the path to the .opf file--hrefs inside it are relative to it.
+    relpath = get_path_part(rootfilename)
+            
+    filecount = 0
+    soups = [] # list of xhmtl blocks
+    images = {} # dict() longdesc->data
+    if getfilecount:
+        # spin through the manifest--only place there are item tags.
+        for item in contentdom.getElementsByTagName("item"):
+            # First, count the 'chapter' files.  FFDL uses file0000.xhtml,
+            # but can also update epubs downloaded from Twisting the
+            # Hellmouth, which uses chapter0.html.
+            if( item.getAttribute("media-type") == "application/xhtml+xml" ):
+                href=relpath+item.getAttribute("href")
+                #print("---- item href:%s path part: %s"%(href,get_path_part(href)))
+                if re.match(r'.*/(file|chapter)\d+\.x?html',href):
+                    if getsoups:
+                        soup = bs.BeautifulSoup(epub.read(href).decode("utf-8"))
+                        for img in soup.findAll('img'):
+                            newsrc=''
+                            longdesc=''
+                            try:
+                                newsrc=get_path_part(href)+img['src']
+                                # remove all .. and the path part above it, if present.
+                                # Most for epubs edited by Sigil.
+                                newsrc = re.sub(r"([^/]+/\.\./)","",newsrc)
+                                longdesc=img['longdesc']
+                                data = epub.read(newsrc)
+                                images[longdesc] = data
+                                img['src'] = img['longdesc']
+                            except Exception as e:
+                                print("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
+                                print("Exception: %s"%(unicode(e)))
+                                traceback.print_exc()
+                        soup = soup.find('body')
+                        # ffdl epubs have chapter title h3
+                        h3 = soup.find('h3')
+                        if h3:
+                            h3.extract()
+                        # TtH epubs have chapter title h2
+                        h2 = soup.find('h2')
+                        if h2:
+                            h2.extract()
+                            
+                        soups.append(soup)
+                        
+                    filecount+=1
+
+    for k in images.keys():
+        print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
+    return (source,filecount,soups,images)
+
+def get_path_part(n):
+    relpath = os.path.dirname(n)
+    if( len(relpath) > 0 ):
+        relpath=relpath+"/"
+    return relpath

fanficdownloader/exceptions.py

@@ -34,12 +34,16 @@ class InvalidStoryURL(Exception):
         return "Bad Story URL: (%s) for site: (%s) Example: (%s)" % (self.url, self.domain, self.example)
 
 class FailedToLogin(Exception):
-    def __init__(self,url,username):
+    def __init__(self,url, username, passwdonly=False):
         self.url=url
         self.username=username
+        self.passwdonly=passwdonly
         
     def __str__(self):
-        return "Failed to Login for URL: (%s) with username: (%s)" % (self.url, self.username)
+        if self.passwdonly:
+            return "URL Failed, password required: (%s) " % (self.url)
+        else:
+            return "Failed to Login for URL: (%s) with username: (%s)" % (self.url, self.username)
 
 class AdultCheckRequired(Exception):
     def __init__(self,url):

fanficdownloader/gziphttp.py

@@ -0,0 +1,38 @@
+## Borrowed from http://techknack.net/python-urllib2-handlers/
+
+import urllib2
+from gzip import GzipFile
+from StringIO import StringIO
+
+class GZipProcessor(urllib2.BaseHandler):
+    """A handler to add gzip capabilities to urllib2 requests
+    """
+    def http_request(self, req):
+        req.add_header("Accept-Encoding", "gzip")
+        return req
+    https_request = http_request
+
+    def http_response(self, req, resp):
+        #print("Content-Encoding:%s"%resp.headers.get("Content-Encoding"))
+        if resp.headers.get("Content-Encoding") == "gzip":
+            gz = GzipFile(
+                        fileobj=StringIO(resp.read()),
+                        mode="r"
+                      )
+#            resp.read = gz.read
+#            resp.readlines = gz.readlines
+#            resp.readline = gz.readline
+#            resp.next = gz.next
+            old_resp = resp
+            resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        return resp
+    https_response = http_response
+
+# brave new world - 1:30 w/o, 1:10 with?  40 chapters, so 20s from sleeps.
+# with gzip, no sleep: 47.469
+# w/o gzip, no sleep: 47.736
+
+# I Am What I Am 67 chapters
+# w/o gzip: 57.168
+# w/ gzip: 40.692

fanficdownloader/htmlcleanup.py

@@ -27,7 +27,7 @@ def _unirepl(match):
     return unichr(value)
 
 def _replaceNumberEntities(data):
-    p = re.compile(r'&#(x?)(\d+);')
+    p = re.compile(r'&#(x?)([0-9a-fA-F]+);')
     return p.sub(_unirepl, data)
 
 def _replaceNotEntities(data):

fanficdownloader/story.py

@@ -15,27 +15,204 @@
 # limitations under the License.
 #
 
-import os
+import os, re
+import urlparse
+from math import floor
 
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
+# Create convert_image method depending on which graphics lib we can
+# load.  Preferred: calibre, PIL, none
+try:
+    from calibre.utils.magick import Image
+
+    def convert_image(url,data,sizes,grayscale):
+        export = False
+        img = Image()
+        img.load(data)
+        
+        owidth, oheight = img.size
+        nwidth, nheight = sizes
+        scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
+        if scaled:
+            img.size = (nwidth, nheight)
+            export = True
+            
+        if grayscale and img.type != "GrayscaleType":
+            img.type = "GrayscaleType"
+            export = True
+
+        if normalize_format_name(img.format) != "jpg":
+            export = True
+
+        if export:
+            return (img.export('JPG'),'jpg','image/jpeg')
+        else:
+            print("image used unchanged")
+            return (data,'jpg','image/jpeg')
+        
+except:
+
+    # No calibre routines, try for PIL for CLI.
+    try:
+        import Image
+        from StringIO import StringIO
+        def convert_image(url,data,sizes,grayscale):
+            
+            export = False
+            img = Image.open(StringIO(data))
+            
+            owidth, oheight = img.size
+            nwidth, nheight = sizes
+            scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
+            if scaled:
+                img = img.resize((nwidth, nheight),Image.ANTIALIAS)
+                export = True
+                
+            if grayscale and img.mode != "L":
+                img = img.convert("L")
+                export = True
+
+            if normalize_format_name(img.format) != "jpg":
+                export = True
+
+            if export:
+                outsio = StringIO()
+                img.save(outsio,'JPEG')
+                return (outsio.getvalue(),'jpg','image/jpeg')
+            else:
+                print("image used unchanged")
+                return (data,'jpg','image/jpeg')
+        
+    except:
+
+        # No calibre or PIL, simple pass through with mimetype.
+        imagetypes = {
+            'jpg':'image/jpeg',
+            'jpeg':'image/jpeg',
+            'png':'image/png',
+            'gif':'image/gif',
+            'svg':'image/svg+xml',
+            }
+
+        def convert_image(url,data,sizes,grayscale):
+            ext=url[url.rfind('.')+1:].lower()
+            return (data,ext,imagetypes[ext])
+        
+def normalize_format_name(fmt):
+    if fmt:
+        fmt = fmt.lower()
+        if fmt == 'jpeg':
+            fmt = 'jpg'
+    return fmt
+
+def fit_image(width, height, pwidth, pheight):
+    '''
+    Fit image in box of width pwidth and height pheight.
+    @param width: Width of image
+    @param height: Height of image
+    @param pwidth: Width of box
+    @param pheight: Height of box
+    @return: scaled, new_width, new_height. scaled is True iff new_width and/or new_height is different from width or height.
+    '''
+    scaled = height > pheight or width > pwidth
+    if height > pheight:
+        corrf = pheight/float(height)
+        width, height = floor(corrf*width), pheight
+    if width > pwidth:
+        corrf = pwidth/float(width)
+        width, height = pwidth, floor(corrf*height)
+    if height > pheight:
+        corrf = pheight/float(height)
+        width, height = floor(corrf*width), pheight
+
+    return scaled, int(width), int(height)
+
+try:
+    # doesn't really matter what, just checking for appengine.
+    from google.appengine.api import apiproxy_stub_map
+
+    is_appengine = True
+except:
+    is_appengine = False
+
+
+# The list comes from ffnet, the only multi-language site we support
+# at the time of writing.  Values are taken largely from pycountry,
+# but with some corrections and guesses.
+langs = {
+    "English":"en",
+    "Spanish":"es",
+    "French":"fr",
+    "German":"de",
+    "Chinese":"zh",
+    "Japanese":"ja",
+    "Dutch":"nl",
+    "Portuguese":"pt",
+    "Russian":"ru",
+    "Italian":"it",
+    "Bulgarian":"bg",
+    "Polish":"pl",
+    "Hungarian":"hu",
+    "Hebrew":"he",
+    "Arabic":"ar",
+    "Swedish":"sv",
+    "Norwegian":"no",
+    "Danish":"da",
+    "Finnish":"fi",
+    "Filipino":"fil",
+    "Esperanto":"eo",
+    "Hindi":"hi",
+    "Punjabi":"pa",
+    "Farsi":"fa",
+    "Greek":"el",
+    "Romanian":"ro",
+    "Albanian":"sq",
+    "Serbian":"sr",
+    "Turkish":"tr",
+    "Czech":"cs",
+    "Indonesian":"id",
+    "Croatian":"hr",
+    "Catalan":"ca",
+    "Latin":"la",
+    "Korean":"ko",
+    "Vietnamese":"vi",
+    "Thai":"th",
+    "Devanagari":"hi",
+    }
+
 class Story:
     
     def __init__(self):
         try:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
-            self.metadata = {'version':'4.2'}
+            self.metadata = {'version':'4.4'}
+        self.replacements = []
         self.chapters = [] # chapters will be tuples of (title,html)
+        self.imgurls = []
+        self.imgtuples = []
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
+        self.cover=None
 
     def setMetadata(self, key, value):
         ## still keeps < < and &
         self.metadata[key]=conditionalRemoveEntities(value)
+        if key == "language":
+            try:
+                self.metadata['langcode'] = langs[self.metadata[key]]
+            except:
+                self.metadata['langcode'] = 'en'
 
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
             return self.metadata[key]
+
+    def doReplacments(self,value):
+        for (p,v) in self.replacements:
+            if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
+                value = re.sub(p,v,value)                
+        return value;
         
     def getMetadata(self, key, removeallentities=False):
         value = None
@@ -50,21 +227,22 @@ class Story:
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
                  value = value.strftime("%Y-%m-%d")
-                 
+
+        value=self.doReplacments(value)
         if removeallentities and value != None:
             return removeAllEntities(value)
         else:
             return value
 
-    def getAllMetadata(self):
+    def getAllMetadata(self, removeallentities=False):
         '''
         All single value *and* list value metadata as strings.
         '''
         allmetadata = {}
         for k in self.metadata.keys():
-            allmetadata[k] = self.getMetadata(k)
+            allmetadata[k] = self.getMetadata(k, removeallentities)
         for l in self.listables.keys():
-            allmetadata[l] = self.getMetadata(l)
+            allmetadata[l] = self.getMetadata(l, removeallentities)
 
         return allmetadata
         
@@ -81,10 +259,14 @@ class Story:
     def getList(self,listname):
         if not self.listables.has_key(listname):
             return []
-        return self.listables[listname]
+        return filter( lambda x : x!=None and x!='' ,
+                       map(self.doReplacments,self.listables[listname]) )
 
     def getLists(self):
-        return self.listables
+        lsts = {}
+        for ln in self.listables.keys():
+            lsts[ln] = self.getList(ln)
+        return lsts
     
     def addChapter(self, title, html):
         self.chapters.append( (title,html) )
@@ -92,10 +274,99 @@ class Story:
     def getChapters(self):
         "Chapters will be tuples of (title,html)"
         return self.chapters
+
+    # pass fetch in from adapter in case we need the cookies collected
+    # as well as it's a base_story class method.
+    def addImgUrl(self,configurable,parenturl,url,fetch,cover=False):
+        
+        url = url.strip() # ran across an image with a space in the
+                          # src. Browser handled it, so we'd better, too.
+
+        # appengine (web version) isn't allowed to do images--just
+        # gets too big too fast and breaks things.
+        if is_appengine:
+            return
+        
+        if url.startswith("http") or url.startswith("file") or parenturl == None:
+            imgurl = url
+        else:
+            parsedUrl = urlparse.urlparse(parenturl)
+            if url.startswith("/") :
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     parsedUrl.netloc,
+                     url,
+                     '','',''))
+            else:
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     parsedUrl.netloc,
+                     parsedUrl.path + url,
+                     '','',''))
+
+        prefix='ffdl'
+        if imgurl not in self.imgurls:
+            parsedUrl = urlparse.urlparse(imgurl)
+            sizes = [ int(x) for x in configurable.getConfigList('image_max_size') ]
+            try:
+                (data,ext,mime) = convert_image(imgurl,
+                                                fetch(imgurl),
+                                                sizes,
+                                                configurable.getConfig('grayscale_images'))
+            except Exception, e:
+                print("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
+                return "failedtoload"
+            
+            # explicit cover, make the first image.
+            if cover and not configurable.getConfig('never_make_cover'):
+                if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
+                    # remove existing cover, if there is one.
+                    del self.imgurls[0]
+                    del self.imgtuples[0]
+                self.imgurls.insert(0,imgurl)
+                newsrc = "images/cover.%s"%ext
+                self.cover=newsrc
+                self.imgtuples.insert(0,{'newsrc':newsrc,'mime':mime,'data':data})
+            else:
+                self.imgurls.append(imgurl)
+                # First image, copy not link because calibre will replace with it's cover.
+                if len(self.imgurls)==1 and \
+                        configurable.getConfig('make_firstimage_cover') and \
+                        not configurable.getConfig('never_make_cover'):
+                    newsrc = "images/cover.%s"%ext
+                    self.cover=newsrc
+                    self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
+                    self.imgurls.append(imgurl)
+            
+                newsrc = "images/%s-%s.%s"%(
+                    prefix,
+                    self.imgurls.index(imgurl),
+                    ext)
+                self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
+                
+            print("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
+        else:
+            newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
+            
+        #print("===============\n%s\nimg url:%s\n============"%(newsrc,self.imgurls[-1]))
+        
+        return newsrc
+
+    def getImgUrls(self):
+        retlist = []
+        for i, url in enumerate(self.imgurls):
+            #parsedUrl = urlparse.urlparse(url)
+            retlist.append(self.imgtuples[i])
+        return retlist
     
     def __str__(self):
         return "Metadata: " +str(self.metadata) + "\nListables: " +str(self.listables) #+ "\nChapters: "+str(self.chapters)
 
+    def setReplace(self,replace):
+        for line in replace.splitlines():
+            if "=>" in line:
+                self.replacements.append(map( lambda x: x.strip(), line.split("=>") ))
+    
 def commaGroups(s):
     groups = []
     while s and s[-1].isdigit():

fanficdownloader/translit.py

@@ -0,0 +1,57 @@
+#-*-coding:utf-8-*-
+# Code taken from http://python.su/forum/viewtopic.php?pid=66946
+import unicodedata
+def is_syllable(letter):
+    syllables = ("A", "E", "I", "O", "U", "a", "e", "i", "o", "u")
+    if letter in syllables:
+        return True
+    return False
+def is_consonant(letter):
+    return not is_syllable(letter)
+def romanize(letter):
+    try:
+        str(letter)
+    except UnicodeEncodeError:
+        pass
+    else:
+        return str(letter)
+    unid = unicodedata.name(letter)
+    exceptions = {"NUMERO SIGN": "No", "LEFT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "DASH": "-"}
+    for name_contains in exceptions:
+        if unid.find(name_contains)!=-1:
+            return exceptions[name_contains]
+    assert(unid.startswith("CYRILLIC"))# Not ready to romanize anything but cyrillics
+    transformation_pairs = {"CYRILLIC CAPITAL LETTER ": str.capitalize, "CYRILLIC SMALL LETTER ": str.lower}
+    func = str.lower
+    for name_contains in transformation_pairs:
+        if unid.find(name_contains)!=-1:
+            func = transformation_pairs[name_contains]
+            unid = unid.replace(name_contains, "")
+    cyrillic_exceptions = {"YERU": "y", "SHORT I": "y", "HARD SIGN": "\'", "SOFT SIGN": "\'", "BYELORUSSIAN-UKRAINIAN I": "i", "GHE WITH UPTURN": "g", "UKRAINIAN IE": "ie", "YU": "yu", "YA": "ya"}
+    for name_contains in cyrillic_exceptions:
+        if unid.find(name_contains)!=-1:
+            return cyrillic_exceptions[name_contains]
+    if all(map(is_syllable, unid)):
+        return func(unid)
+    else:
+        return func(filter(is_consonant, unid))
+def translit(text):
+    output = ""
+    for letter in text:
+        output += romanize(letter)
+    return output
+#def main():
+    #text = u"русск.: Любя, съешь щипцы, — вздохнёт мэр, — кайф жгуч."
+    #print translit(text)
+    #text = u"укр.: Гей, хлопці, не вспію - на ґанку ваша файна їжа знищується бурундучком."
+    #print translit(text)
+    #text = u"болг.: Ах, чудна българска земьо, полюшквай цъфтящи жита."
+    #print translit(text)
+    #text = u"серб.: Неуредне ноћне даме досађивале су Џеку К."
+    #print translit(text)
+    #russk.: Lyubya, s'iesh' shchiptsy, - vzdohniot mer, - kayf zhghuch.
+    #ukr.: Ghiey, hloptsi, nie vspiyu - na ganku vasha fayna yzha znishchuiet'sya burunduchkom.
+    #bolgh.: Ah, chudna b'lgharska ziem'o, polyushkvay ts'ftyashchi zhita.
+    #sierb.: Nieuriednie notshnie damie dosadjivalie su Dzhieku K.
+if __name__=="__main__":
+    main()

fanficdownloader/writers/base_writer.py

@@ -39,17 +39,23 @@ class BaseStoryWriter(Configurable):
 
     def __init__(self, config, adapter):
         Configurable.__init__(self, config)
-        self.addConfigSection(adapter.getSiteDomain())
-        self.addConfigSection(self.getFormatName())
-        self.addConfigSection(adapter.getSiteDomain()+":"+self.getFormatName())
-        self.addConfigSection("overrides")
+        self.setSectionOrder(adapter.getSiteDomain(),self.getFormatName())
+        # self.addConfigSection(adapter.getSiteDomain())
+        # self.addConfigSection(self.getFormatName())
+        # self.addConfigSection(adapter.getSiteDomain()+":"+self.getFormatName())
+        # self.addConfigSection("overrides")
         
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
+        
+        self.story.setReplace(self.getConfig('replace_metadata'))
+        
         self.validEntries = [
             'category',
             'genre',
+            'language',
             'characters',
+            'series',
             'status',
             'datePublished',
             'dateUpdated',
@@ -76,7 +82,9 @@ class BaseStoryWriter(Configurable):
         self.titleLabels = {
             'category':'Category',
             'genre':'Genre',
+            'language':'Language',
             'status':'Status',
+            'series':'Series',
             'characters':'Characters',
             'datePublished':'Published',
             'dateUpdated':'Updated',
@@ -97,10 +105,13 @@ class BaseStoryWriter(Configurable):
             'formatname':'File Format',
             'formatext':'File Extension',
             'siteabbrev':'Site Abbrev',
-            'version':'FFD Version'
+            'version':'FFDL Version'
             }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
+        
+        for tag in self.getConfigList("extratags"):
+            self.story.addToList("extratags",tag)
 
     def getMetadata(self,key):
         return stripHTML(self.story.getMetadata(key))
@@ -134,7 +145,7 @@ class BaseStoryWriter(Configurable):
     def _write(self, out, text):
         out.write(text.encode('utf8'))
 
-    def writeTitlePage(self, out, START, ENTRY, END, WIDE_ENTRY=None):
+    def writeTitlePage(self, out, START, ENTRY, END, WIDE_ENTRY=None, NO_TITLE_ENTRY=None):
         """
         Write the title page, but only include entries that there's
         metadata for.  START, ENTRY and END are expected to already by
@@ -161,6 +172,12 @@ class BaseStoryWriter(Configurable):
                             label=self.getConfig(entry+"_label")
                         else:
                             label=self.titleLabels[entry]
+
+                        # If the label for the title entry is empty, use the
+                        # 'no title' option if there is one.  
+                        if label == "" and NO_TITLE_ENTRY:
+                           TEMPLATE= NO_TITLE_ENTRY
+                           
                         self._write(out,TEMPLATE.substitute({'label':label,
                                                              'value':self.story.getMetadata(entry)}))
 
@@ -184,13 +201,17 @@ class BaseStoryWriter(Configurable):
 
     # if no outstream is given, write to file.
     def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
-        for tag in self.getConfigList("extratags"):
-            self.story.addToList("extratags",tag)
 
         self.metaonly = metaonly
         if outfilename == None:
             outfilename=self.getOutputFileName()
 
+        # minor cheat, tucking css into metadata.
+        if self.getConfig("output_css"):
+            self.story.metadata["output_css"] = self.getConfig("output_css")
+        else:
+            self.story.metadata["output_css"] = ''
+            
         if not outstream:
             close=True
             logging.debug("Save directly to file: %s" % outfilename)
@@ -266,9 +287,6 @@ class BaseStoryWriter(Configurable):
             if name in self.getConfigList("include_subject_tags"):
                 for tag in lst:
                     subjectset.add(tag)
-                    
-        for tag in self.getConfigList("extratags"):
-            subjectset.add(tag)
 
         return list(subjectset)
             

fanficdownloader/writers/writer_epub.py

@@ -20,6 +20,7 @@ import string
 import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
+import urllib
 
 ## XML isn't as forgiving as HTML, so rather than generate as strings,
 ## use DOM to generate the XML files.
@@ -41,32 +42,7 @@ class EpubWriter(BaseStoryWriter):
     def __init__(self, config, story):
         BaseStoryWriter.__init__(self, config, story)
 
-        self.EPUB_CSS = string.Template('''
-body { margin: 2%;
-       text-align: justify;
-       background-color: #${background_color}; }
-pre { font-size: x-small; }
-sml { font-size: small; }
-h1 { text-align: center; }
-h2 { text-align: center; }
-h3 { text-align: center; }
-h4 { text-align: center; }
-h5 { text-align: center; }
-h6 { text-align: center; }
-.CI {
-    text-align:center;
-    margin-top:0px;
-    margin-bottom:0px;
-    padding:0px;
-    }
-.center   {text-align: center;}
-.cover    {text-align: center;}
-.full     {width: 100%; }
-.quarter  {width: 25%; }
-.smcap    {font-variant: small-caps;}
-.u        {text-decoration: underline;}
-.bold     {font-weight: bold;}
-''')
+        self.EPUB_CSS = string.Template('''${output_css}''')
         
         self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -82,6 +58,10 @@ h6 { text-align: center; }
 
         self.EPUB_TITLE_ENTRY = string.Template('''
 <b>${label}:</b> ${value}<br />
+''')
+
+        self.EPUB_NO_TITLE_ENTRY = string.Template('''
+${value}<br />
 ''')
 
         self.EPUB_TITLE_PAGE_END = string.Template('''
@@ -109,6 +89,10 @@ h6 { text-align: center; }
 
         self.EPUB_TABLE_TITLE_WIDE_ENTRY = string.Template('''
 <tr><td colspan="2"><b>${label}:</b> ${value}</td></tr>
+''')
+
+        self.EPUB_TABLE_NO_TITLE_ENTRY = string.Template('''
+<tr><td colspan="2">${label}${value}</td></tr>
 ''')
 
         self.EPUB_TABLE_TITLE_PAGE_END = string.Template('''
@@ -228,7 +212,10 @@ h6 { text-align: center; }
 
         metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
         metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
-        metadata.appendChild(newTag(contentdom,"dc:language",text="en"))
+        if self.story.getMetadata('langcode') != None:
+            metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
+        else:
+            metadata.appendChild(newTag(contentdom,"dc:language",text='en'))
 
         #  published, created, updated, calibre
         #  Leave calling self.story.getMetadataRaw directly in case date format changes.
@@ -274,7 +261,54 @@ h6 { text-align: center; }
         itemrefs = [] # list of strings -- idrefs from .opfs' spines
         items.append(("ncx","toc.ncx","application/x-dtbncx+xml",None)) ## we'll generate the toc.ncx file,
                                                                    ## but it needs to be in the items manifest.
+
+        if self.getConfig('include_images'):
+            imgcount=0
+            for imgmap in self.story.getImgUrls():
+                imgfile = "OEBPS/"+imgmap['newsrc']
+                outputepub.writestr(imgfile,imgmap['data'])
+                items.append(("image%04d"%imgcount,
+                              imgfile,
+                              imgmap['mime'],
+                              None))
+                imgcount+=1
+
+        
         items.append(("style","OEBPS/stylesheet.css","text/css",None))
+
+        guide = None
+        coverIO = None
+
+        if self.story.cover:
+            # Note that the id of the cover xhmtl *must* be 'cover'
+            # for it to work on Nook.
+            items.append(("cover","OEBPS/cover.xhtml","application/xhtml+xml",None))
+            itemrefs.append("cover")
+            # 
+            # <meta name="cover" content="cover.jpg"/>
+            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0000",
+                                                           "name":"cover"}))
+            # cover stuff for later:
+            # at end of <package>:
+            # <guide>
+            # <reference type="cover" title="Cover" href="Text/cover.xhtml"/>
+            # </guide>
+            guide = newTag(contentdom,"guide")
+            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
+                                                       "title":"Cover",
+                                                       "href":"OEBPS/cover.xhtml"}))
+            
+            coverIO = StringIO.StringIO()
+            coverIO.write('''
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head><title>Cover</title><style type="text/css" title="override_css">
+@page {padding: 0pt; margin:0pt}
+body { text-align: center; padding:0pt; margin: 0pt; }
+div { margin: 0pt; padding: 0pt; }
+</style></head><body><div>
+<img src="%s" alt="cover"/>
+</div></body></html>
+'''%self.story.cover)
+            
         if self.getConfig("include_titlepage"):
             items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page"))
             itemrefs.append("title_page")
@@ -305,8 +339,18 @@ h6 { text-align: center; }
             spine.appendChild(newTag(contentdom,"itemref",
                                      attrs={"idref":itemref,
                                             "linear":"yes"}))
+        # guide only exists if there's a cover.
+        if guide:
+            package.appendChild(guide)
+            
         # write content.opf to zip.
-        outputepub.writestr("content.opf",contentdom.toxml(encoding='utf-8'))
+        contentxml = contentdom.toxml(encoding='utf-8')
+        
+        # tweak for brain damaged Nook STR.  Nook insists on name before content.
+        contentxml = contentxml.replace('<meta content="image0000" name="cover"/>',
+                                        '<meta name="cover" content="image0000"/>')
+        outputepub.writestr("content.opf",contentxml)
+
         contentdom.unlink()
         del contentdom
 
@@ -342,7 +386,7 @@ h6 { text-align: center; }
         index=0
         for item in items:
             (id,href,type,title)=item
-            # only items to be skipped, toc.ncx, stylesheet.css, should have no title.
+            # only items to be skipped, cover.xhtml, images, toc.ncx, stylesheet.css, should have no title.
             if title :
                 navPoint = newTag(tocncxdom,"navPoint",
                                   attrs={'id':id,
@@ -355,32 +399,39 @@ h6 { text-align: center; }
                 navPoint.appendChild(newTag(tocncxdom,"content",attrs={"src":href}))
                 index=index+1
         
-        # write toc.ncs to zip file
+        # write toc.ncx to zip file
         outputepub.writestr("toc.ncx",tocncxdom.toxml(encoding='utf-8'))
         tocncxdom.unlink()
         del tocncxdom
 
         # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")})) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 
 
         # write title page.
         if self.getConfig("titlepage_use_table"):
             TITLE_PAGE_START  = self.EPUB_TABLE_TITLE_PAGE_START
             TITLE_ENTRY       = self.EPUB_TABLE_TITLE_ENTRY
             WIDE_TITLE_ENTRY  = self.EPUB_TABLE_TITLE_WIDE_ENTRY
+            NO_TITLE_ENTRY    = self.EPUB_TABLE_NO_TITLE_ENTRY
             TITLE_PAGE_END    = self.EPUB_TABLE_TITLE_PAGE_END
         else:
             TITLE_PAGE_START  = self.EPUB_TITLE_PAGE_START
             TITLE_ENTRY       = self.EPUB_TITLE_ENTRY
             WIDE_TITLE_ENTRY  = self.EPUB_TITLE_ENTRY # same, only wide in tables.
+            NO_TITLE_ENTRY    = self.EPUB_NO_TITLE_ENTRY
             TITLE_PAGE_END    = self.EPUB_TITLE_PAGE_END
-        
+
+        if coverIO:
+            outputepub.writestr("OEBPS/cover.xhtml",coverIO.getvalue())
+            coverIO.close()
+            
         titlepageIO = StringIO.StringIO()
         self.writeTitlePage(out=titlepageIO,
                             START=TITLE_PAGE_START,
                             ENTRY=TITLE_ENTRY,
                             WIDE_ENTRY=WIDE_TITLE_ENTRY,
-                            END=TITLE_PAGE_END)
+                            END=TITLE_PAGE_END,
+                            NO_TITLE_ENTRY=NO_TITLE_ENTRY)
         if titlepageIO.getvalue(): # will be false if no title page.
             outputepub.writestr("OEBPS/title_page.xhtml",titlepageIO.getvalue())
         titlepageIO.close()
@@ -406,7 +457,7 @@ h6 { text-align: center; }
                 fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                 del fullhtml
- 
+
 	# declares all the files created by Windows.  otherwise, when
         # it runs in appengine, windows unzips the files as 000 perms.
         for zf in outputepub.filelist:
@@ -424,4 +475,4 @@ def newTag(dom,name,attrs=None,text=None):
     if( text is not None ):
         tag.appendChild(dom.createTextNode(text))
     return tag
-    
+

fanficdownloader/writers/writer_html.py

@@ -39,20 +39,7 @@ class HTMLWriter(BaseStoryWriter):
 <head>
 <title>${title} by ${author}</title>
 <style type="text/css">
-body { background-color: #${background_color}; }
-.CI {
-    text-align:center;
-    margin-top:0px;
-    margin-bottom:0px;
-    padding:0px;
-    }
-.center   {text-align: center;}
-.cover    {text-align: center;}
-.full     {width: 100%; }
-.quarter  {width: 25%; }
-.smcap    {font-variant: small-caps;}
-.u        {text-decoration: underline;}
-.bold     {font-weight: bold;}
+${output_css}
 </style>
 </head>
 <body>
@@ -95,9 +82,6 @@ body { background-color: #${background_color}; }
 
     def writeStoryImpl(self, out):
 
-        # minor cheat, tucking bg into metadata.
-        if self.getConfig("background_color"):
-            self.story.metadata["background_color"] = self.getConfig("background_color")
         self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
 
         self.writeTitlePage(out,

fanficdownloader/writers/writer_mobi.py

@@ -41,7 +41,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -50,6 +49,10 @@ class MobiWriter(BaseStoryWriter):
 
         self.MOBI_TITLE_ENTRY = string.Template('''
 <b>${label}:</b> ${value}<br />
+''')
+
+        self.MOBI_NO_TITLE_ENTRY = string.Template('''
+${value}<br />
 ''')
 
         self.MOBI_TITLE_PAGE_END = string.Template('''
@@ -64,7 +67,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -77,6 +79,10 @@ class MobiWriter(BaseStoryWriter):
 
         self.MOBI_TABLE_TITLE_WIDE_ENTRY = string.Template('''
 <tr><td colspan="2"><b>${label}:</b> ${value}</td></tr>
+''')
+
+        self.MOBI_TABLE_NO_TITLE_WIDE_ENTRY = string.Template('''
+<tr><td colspan="2">${value}</td></tr>
 ''')
 
         self.MOBI_TABLE_TITLE_PAGE_END = string.Template('''
@@ -91,7 +97,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <div>
@@ -113,7 +118,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${chapter}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3>${chapter}</h3>
@@ -133,11 +137,13 @@ class MobiWriter(BaseStoryWriter):
             TITLE_PAGE_START  = self.MOBI_TABLE_TITLE_PAGE_START
             TITLE_ENTRY       = self.MOBI_TABLE_TITLE_ENTRY
             WIDE_TITLE_ENTRY  = self.MOBI_TABLE_TITLE_WIDE_ENTRY
+            NO_TITLE_ENTRY    = self.MOBI_TABLE_NO_TITLE_ENTRY
             TITLE_PAGE_END    = self.MOBI_TABLE_TITLE_PAGE_END
         else:
             TITLE_PAGE_START  = self.MOBI_TITLE_PAGE_START
             TITLE_ENTRY       = self.MOBI_TITLE_ENTRY
             WIDE_TITLE_ENTRY  = self.MOBI_TITLE_ENTRY # same, only wide in tables.
+            NO_TITLE_ENTRY    = self.MOBI_NO_TITLE_ENTRY
             TITLE_PAGE_END    = self.MOBI_TITLE_PAGE_END
         
         titlepageIO = StringIO.StringIO()
@@ -145,7 +151,8 @@ class MobiWriter(BaseStoryWriter):
                             START=TITLE_PAGE_START,
                             ENTRY=TITLE_ENTRY,
                             WIDE_ENTRY=WIDE_TITLE_ENTRY,
-                            END=TITLE_PAGE_END)
+                            END=TITLE_PAGE_END,
+                            NO_TITLE_ENTRY=NO_TITLE_ENTRY)
         if titlepageIO.getvalue(): # will be false if no title page.
             files.append(titlepageIO.getvalue())
         titlepageIO.close()

index-ajax.html

@@ -4,7 +4,7 @@
 		<link href="css/index.css" rel="stylesheet" type="text/css">
 		<link type="text/css" href="http://jqueryui.com/latest/themes/base/ui.all.css" rel="stylesheet" />
 	 
-		<title>Fanfiction Downloader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
+		<title>FanFictionDownLoader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
 		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
 		<script src="/js/jquery-1.3.2.js"></script>
 		<script src="/js/fdownloader.js"></script>
@@ -16,7 +16,7 @@
 	<body>
 		<div id='main'>
 			<h1>
-				<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+				<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
 			</h1>
 				
 <!--			<form action="/fdown" method="post"> -->
@@ -91,8 +91,8 @@
 					<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 					alt="Powered by Google App Engine" />
 					<br/><br/>
-					FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-					Copyright &copy; <a href="http://twitter.com/sigizmund">Roman Kirillov</a>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; Fanficdownloader team
 				</div>
 <!--			</form> -->
 		</div>

index.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -26,7 +26,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a> <g:plusone size="medium"></g:plusone>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a> <g:plusone size="medium"></g:plusone>
       </h1>
 
       <div style="text-align: center">
@@ -50,58 +50,39 @@
       <form action="/fdown" method="post">
 	<div id='urlbox'>
 	  <div id='greeting'>
-	    <p>Hi, {{ nickname }}! This is a fan fiction downloader, which makes reading stories from various websites
+	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>This is the Official Multithreading Version</h3>
+	  <h3>Outages</h3>
 	  <p>
-	    This version of the application uses Python 2.7 and
-	    multithreading to try and reduce our usage.  Google
-	    considers Python 2.7 Experimental still, so there may be issues.
+	    This web service has been having almost daily outages
+	    recently due to problems with running out of quota for the
+	    application.  We are working to improve the situation.
+	  </p>
+	  <h3>Update - New Sites</h3>
+	  <p>
+	    Four new sites have been added:
+	    <ul>
+	      <li>fanfiction.mugglenet.com</li>
+	      <li>fanfiction.portkey.org</li>
+	      <li>thequidditchpitch.org</li>
+	      <li>www.hpfandom.net</li>
+	    </ul>
+	    These are the four largest/most active of the sites that have been requested lately.
 	  </p>
 	  <p>
-	    <b>FanFictionDownLoader calibre Plugin</b>
-	    <br /><br />
-	    
-	    There's now a version of this downloader that runs
-	    entirely inside the
-	    popular <a href="http://calibre-ebook.com/">calibre</a>
-	    ebook management package as a plugin.
-	    
-	    <br /><br />
-	    
-	    Once you have calibre installed and running, inside
-	    calibre, you can go to 'Get plugins to enhance calibre' or
-	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
-	    
-	  </p>
-	  <p>
-	    <b>Wider support for 'Characters'</b>
-	    <br /><br />
-	    We now collect 'Character' lists for most of the supported sites.  
-	    <br /><br />
-	    There's huge variation in how different sites choose to define characters--well beyond our ability to standardize.
-	    <br /><br />
-	    So if you don't like and don't want to see characters in your title pages or tags, you can turn them off in your configuration with:
-	    <pre>
-[defaults]
-# note that characters is removed.  This removes it from your title page.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
-
-# note that characters is removed.  This removes it from your tags.
-include_subject_tags: extratags, genre, category, status
-	    </pre>
-	    If you like them for some sites, but not others, just copy the titlepage_entries and include_subject_tags to sections for different sites and edit to taste.
+	    Another site that's been requested a few times is
+	    adultfanfiction.net.  None of the current developers are
+	    interested in writing code to support it because of the
+	    ominous legal warnings and requirements to access the
+	    site.
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
-	    the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.  The
-	    <a href="http://4-2-0.fanfictiondownloader.appspot.com">Previous
-	      Version</a> is also available for you to use if necessary.
+	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
+	    <a href="http://4-4-0.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -137,14 +118,33 @@ include_subject_tags: extratags, genre, category, status
       <div id='urlbox'>
 	<div id='greeting'>
 	  <p>
-	    This is a fan fiction downloader, which makes reading stories from various websites much easier. Before you
-	    can start downloading fanfics, you need to login, so downloader can remember your fanfics and store them.
+	    This is a FanFictionDownLoader, which makes reading stories from various websites much easier. Before you
+	    can start downloading fanfics, you need to login, so FanFictionDownLoader can remember your fanfics and store them.
 	  </p>
 	  <p><a href="{{ login_url }}">Login using Google account</a></p>
 	</div>
       </div>
       {% endif %}
       
+      <div id='typebox'>
+	  <p>
+	    <b>FanFictionDownLoader calibre Plugin</b>
+	    <br /><br />
+	    
+	    There's now a version of this downloader that runs
+	    entirely inside the
+	    popular <a href="http://calibre-ebook.com/">calibre</a>
+	    ebook management package as a plugin.
+	    
+	    <br /><br />
+	    
+	    Once you have calibre installed and running, inside
+	    calibre, you can go to 'Get plugins to enhance calibre' or
+	    'Get new plugins' and
+	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
+	    
+	  </p>
+      </div>
       <div id='helpbox'>
 	<dl>
 	  <dt>fictionalley.org</dt>
@@ -245,10 +245,48 @@ include_subject_tags: extratags, genre, category, status
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://www.siye.co.uk/siye/viewstory.php?sid=123">http://www.siye.co.uk/siye/viewstory.php?sid=123</a>.
 	  </dd>
-	</dl>	
-
-	
-	A few additional things to know, which will make your life substantially easier:
+	  <dt>archiveofourown.org</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://archiveofourown.org/works/76366">http://archiveofourown.org/works/76366</a>.
+	    <br /><a href="http://archiveofourown.org/works/76366/chapters/101584">http://archiveofourown.org/works/76366/chapters/101584</a>.
+	  </dd>
+	  <dt>ficbook.net(Russian)</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://ficbook.net/readfic/93626">http://ficbook.net/readfic/93626</a>.
+	    <br /><a href="http://ficbook.net/readfic/93626/246417#part_content">http://ficbook.net/readfic/93626/246417#part_content</a>.
+	  </dd>
+	  <dt>gayauthors.org</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://www.gayauthors.org/story/mark-arbour/stvincent">http://www.gayauthors.org/story/mark-arbour/stvincent</a>.
+	    <br /><a href="http://www.gayauthors.org/story/Mark%20Arbour/stvincent/7">http://www.gayauthors.org/story/Mark Arbour/stvincent/7</a>.
+	  </dd>
+	  <dt>fanfiction.mugglenet.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://fanfiction.mugglenet.com/viewstory.php?sid=123">http://fanfiction.mugglenet.com/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>www.hpfandom.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.hpfandom.net/eff/viewstory.php?sid=123">http://www.hpfandom.net/eff/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>thequidditchpitch.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://thequidditchpitch.org/viewstory.php?sid=123">http://thequidditchpitch.org/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>fanfiction.portkey.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://fanfiction.portkey.org/story/123">http://fanfiction.portkey.org/story/123</a>.
+	  </dd>
+	</dl>
+	<p>
+	  A few additional things to know, which will make your life substantially easier:
+	</p>
 	<ol>
 	  <li>
 	    First thing to know: I do not use your Google login and password. In fact, all I know about it is your ID &ndash; password
@@ -281,8 +319,8 @@ include_subject_tags: extratags, genre, category, status
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
       </div>
 
       <div style="margin-top: 1em; text-align: center'">

login.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Login Needed Fanfiction Downloader</title>
+    <title>Login Needed FanFictionDownLoader</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -22,7 +22,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
 
       <div style="text-align: center">
@@ -88,8 +88,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
       </div>
 
       <div style="margin-top: 1em; text-align: center'">

main.py

@@ -339,7 +339,7 @@ class FanfictionDownloader(UserConfigServer):
                 self.redirect("/?error=custom&errtext=%s"%urlEscape("There's an error in your User Configuration: "+str(e)))
                 return
 
-            adapter = adapters.getAdapter(config,url)
+            adapter = adapters.getAdapter(config,url,format)
             logging.info('Created an adaper: %s' % adapter)
 
             if len(login) > 1:
@@ -442,7 +442,7 @@ class FanfictionDownloaderTask(UserConfigServer):
 
         try:
             config = self.getUserConfig(user)
-            adapter = adapters.getAdapter(config,url)
+            adapter = adapters.getAdapter(config,url,format)
 
             logging.info('Created an adapter: %s' % adapter)
 

plugin-defaults.ini

@@ -41,7 +41,9 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
+language_label:Language
 characters_label:Characters
+series_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -71,7 +73,14 @@ version_label:FFDL Version
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
 
 ## include title page as first page.
 include_titlepage: true
@@ -99,16 +108,55 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
-## output background color--only used by html and epub (and ignored in
-## epub by many readers).  Must be hex code, # will be added.
-background_color: ffffff
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
+
+## Some readers don't show horizontal rule (<hr />) tags correctly.
+## This replaces them all with a centered '* * *'.  (Note centering
+## doesn't work on some devices either.)
+#replace_hr: false
 
 ## Each output format has a section that overrides [defaults]
 [html]
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css:
+ body { background-color: #%(background_color)s; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -125,11 +173,86 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css: 
+ body { background-color: #%(background_color)s;
+        text-align: justify;
+        margin: 2%%; }
+ pre { font-size: x-small; }
+ sml { font-size: small; }
+ h1 { text-align: center; }
+ h2 { text-align: center; }
+ h3 { text-align: center; }
+ h4 { text-align: center; }
+ h5 { text-align: center; }
+ h6 { text-align: center; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
+## include images from img tags in the body and summary of
+## stories.  Images will be converted to jpg for size if possible.
+#include_images:false
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:false
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover: false
+
+## If set, the epub will never have a cover, even include_images is on
+## and the site has specific cover images.
+#never_make_cover: false
+
+## If set, and there isn't already a cover image from the adapter or
+## from make_firstimage_cover, this image will be made the cover.
+## It can be either a 'file:' or 'http:' url.
+## Note that if you enable make_firstimage_cover in [epub], but want
+## to use default_cover_image for a specific site, use the site:format
+## section, for example: [www.ficwad.com:epub]
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/icon.png
+#default_cover_image:http://www.somesite.com/someimage.gif
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+## if the <img> tag doesn't have a div or a p around it, nook gets
+## confused and displays it on every page after that under the text
+## for the rest of the chapter.  I doubt adding a div around the img
+## will break any other readers, but in case it does, the fix can be
+## turned off.
+nook_img_fix:true
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
-
 ## Each site has a section that overrides [defaults] *and* the format
 ## sections test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
@@ -146,8 +269,79 @@ extratags: FanFiction,Testing,Text
 [test1.com:html]
 extratags: FanFiction,Testing,HTML
 
+[castlefans.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.mugglenet.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.tenhawkpresents.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[thequidditchpitch.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.archiveofourown.org]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.fanfiction.net]
 
+[www.ficbook.net]
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.fictionpress.com]
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
@@ -160,26 +354,37 @@ extratags:
 #username:YourName
 #password:yourpassword
 
-[www.twilighted.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
 
-[www.twiwrite.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
+## fimfiction.net stories can be locked requiring individual
+## passwords.  If fail_on_password is set, the downloader will fail
+## when a password is required rather than prompting every time.
+#fail_on_password: false
 
-[www.whofic.com]
+[www.gayauthors.org]
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.hpfandom.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
 
 [www.mediaminer.org]
 
+[www.potionsandsnitches.net]
+
+[www.siye.co.uk]
+
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -193,7 +398,23 @@ extratags:
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-[www.ficwad.com]
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
+[www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -201,42 +422,22 @@ extratags:
 #username:YourName
 #password:yourpassword
 
-[www.adastrafanfic.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
 
-[www.fictionalley.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.harrypotterfanfiction.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.fimfiction.net]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.tthfanfic.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-## tth is a little unusual--it doesn't require user/pass, but the site
-## keeps track of which chapters you've read and won't send another
-## update until it thinks you're up to date.  This way, on download,
-## it thinks you're up to date.
+[www.twiwrite.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
 #username:YourName
 #password:yourpassword
 
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.whofic.com]
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

plugin-example.ini

@@ -7,11 +7,45 @@
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## include images from img tags in the body and summary of
+## stories.  Images will be converted to jpg for size if possible.
+#include_images:true
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:true
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover:true
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+#image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+## Turn off in [defaults] or [overrides] to prevent all sites from
+## updating series column.
+## default is true
+#collect_series: false
+
 ## Most common, I expect will be using this to save username/passwords
 ## for different sites.
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
+## default is false
+#collect_series: true
 
 [www.ficwad.com]
 #username:YourUsername
@@ -20,6 +54,8 @@
 [www.twiwrite.net]
 #username:YourName
 #password:yourpassword
+## default is false
+#collect_series: true
 
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
@@ -30,6 +66,8 @@
 #username:YourName
 #password:yourpassword
 #is_adult:true
+## default is false
+#collect_series: true
 
 [www.fictionalley.org]
 #is_adult:true
@@ -39,6 +77,7 @@
 
 [www.fimfiction.net]
 #is_adult:true
+#fail_on_password: false
 
 [www.tthfanfic.org]
 #is_adult:true
@@ -53,3 +92,6 @@
 ## This section will override anything in the system defaults or other
 ## sections here.
 [overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true

recent.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
+    <title>FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <script type="text/javascript">
 
@@ -21,7 +21,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
       
       <script type="text/javascript"><!--
@@ -46,26 +46,6 @@
 	</div>
       </div>
 
-      <div id='typebox'>
-	  <p>
-	    <b>FanFictionDownLoader calibre Plugin</b>
-	    <br /><br />
-	    
-	    There's now a version of this downloader that runs
-	    entirely inside the
-	    popular <a href="http://calibre-ebook.com/">calibre</a>
-	    ebook management package as a plugin.
-	    
-	    <br /><br />
-	    
-	    Once you have calibre installed and running, inside
-	    calibre, you can go to 'Get plugins to enhance calibre' or
-	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
-	    
-	  </p>
-      </div>
-      
       <div id='helpbox'>
 	{% for fic in fics %}
 	<p>

status.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>{% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - Fanfiction Downloader</title>
+    <title>{% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - FanFictionDownLoader</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     {% if not fic.completed and not fic.failure %}
@@ -25,7 +25,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
       <div style="text-align: center">
         <script type="text/javascript"><!--
@@ -68,30 +68,11 @@
 	  <p>See your personal list of <a href="/recent">previously downloaded fanfics</a>.</p>
       </div>
 
-      <div id='typebox'>
-	  <p>
-	    <b>FanFictionDownLoader calibre Plugin</b>
-	    <br /><br />
-	    
-	    There's now a version of this downloader that runs
-	    entirely inside the
-	    popular <a href="http://calibre-ebook.com/">calibre</a>
-	    ebook management package as a plugin.
-	    
-	    <br /><br />
-	    
-	    Once you have calibre installed and running, inside
-	    calibre, you can go to 'Get plugins to enhance calibre' or
-	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
-	    
-	  </p>
-      </div>
       <div style='text-align: center'>
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
       </div>