Bump versions.

AO3 doesn't reliably report chapter author.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-23
+version: 4-4-51
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 6, 7)
+    version             = (1, 7, 18)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code
@@ -80,11 +80,29 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
         if ac is not None:
             ac.apply_settings()
 
-# For testing, run from command line with this:
-# calibre-debug -e __init__.py
-# 
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    from calibre.gui2.preferences import test_widget
-    app = QApplication([])
-    test_widget('Advanced', 'Plugins')
+    def cli_main(self,argv):
+        # I believe there's no performance hit loading these here when
+        # CLI--it would load everytime anyway.
+        from StringIO import StringIO
+        from calibre.library import db
+        from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
+        from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
+        from calibre.utils.config import prefs as calibre_prefs
+        from optparse import OptionParser      
+
+        parser = OptionParser('%prog --run-plugin '+self.name+' -- [options] <storyurl>')
+        parser.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
+        # parser.add_option('--dont-notify-gui', default=False, action='store_true',
+        #               help=_('Do not notify the running calibre GUI (if any) that the database has'
+        #                      ' changed. Use with care, as it can lead to database corruption!'))
+
+        pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
+                 or not x.startswith('-')]
+        opts, args = parser.parse_args(pargs)
+
+        ffdl_prefs = PrefsFacade(db(path=opts.library_path,
+                                    read_only=True))
+        ffdl_main(argv[1:],
+                  parser=parser,
+                  passed_defaultsini=StringIO(get_resources("defaults.ini")),
+                  passed_personalini=StringIO(ffdl_prefs["personal.ini"]))

calibre-plugin/common_utils.py

@@ -196,7 +196,7 @@ class ImageTitleLayout(QHBoxLayout):
     '''
     A reusable layout widget displaying an image followed by a title
     '''
-    def __init__(self, parent, icon_name, title):
+    def __init__(self, parent, icon_name, title, tooltip=None):
         QHBoxLayout.__init__(self)
         title_image_label = QLabel(parent)
         pixmap = get_pixmap(icon_name)
@@ -217,6 +217,9 @@ class ImageTitleLayout(QHBoxLayout):
         self.addWidget(shelf_label)
         self.insertStretch(-1)
 
+        if tooltip:
+            title_image_label.setToolTip(tooltip)
+            shelf_label.setToolTip(tooltip)
 
 class SizePersistedDialog(QDialog):
     '''

calibre-plugin/config.py

@@ -7,138 +7,111 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback, copy
+import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
+                      QDialogButtonBox )
 
-from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
 
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order)
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
+    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
+            EditTextDialog, RejectUrlEntry)
+    
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
+    import (getConfigSections, getNormalStoryURL)
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
+    import ( KeyboardConfigDialog, PrefsViewerDialog )
 
 from calibre.gui2.complete import MultiCompleteLineEdit
 
-PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
-PREFS_KEY_SETTINGS = 'settings'
+class RejectURLList:
+    def __init__(self,prefs):
+        self.prefs = prefs
+        self.sync_lock = threading.RLock()
+        self.listcache = None
 
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-default_prefs = {}
-default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+    def _read_list_from_text(self,text,addreasontext=''):
+        cache = OrderedDict()
 
-default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
-default_prefs['updateepubcover'] = False
-default_prefs['keeptags'] = False
-default_prefs['urlsfromclip'] = True
-default_prefs['updatedefault'] = True
-default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = OVERWRITE
-default_prefs['deleteotherforms'] = False
-default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
-default_prefs['lookforurlinhtml'] = False
-default_prefs['injectseries'] = False
+        #print("_read_list_from_text")
+        for line in text.splitlines():
+            rue = RejectUrlEntry(line,addreasontext=addreasontext,fromline=True)
+            #print("rue.url:%s"%rue.url)
+            if rue.valid:
+                cache[rue.url] = rue
+        return cache        
 
-default_prefs['send_lists'] = ''
-default_prefs['read_lists'] = ''
-default_prefs['addtolists'] = False
-default_prefs['addtoreadlists'] = False
-default_prefs['addtolistsonread'] = False
+    def _get_listcache(self):
+        if self.listcache == None:
+            self.listcache = self._read_list_from_text(prefs['rejecturls'])
+        return self.listcache
 
-default_prefs['gcnewonly'] = False
-default_prefs['gc_site_settings'] = {}
-default_prefs['allow_gc_from_ini'] = True
-
-default_prefs['countpagesstats'] = []
-
-default_prefs['errorcol'] = ''
-default_prefs['custom_cols'] = {}
-default_prefs['custom_cols_newonly'] = {}
-
-default_prefs['std_cols_newonly'] = {}
-
-def set_library_config(library_config):
-    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
-                                              PREFS_KEY_SETTINGS,
-                                              library_config)
-    
-def get_library_config():
-    db = get_gui().current_db
-    library_id = get_library_uuid(db)
-    library_config = None
-    # Check whether this is a configuration needing to be migrated
-    # from json into database.  If so: get it, set it, rename it in json.
-    if library_id in old_prefs:
-        #print("get prefs from old_prefs")
-        library_config = old_prefs[library_id]
-        set_library_config(library_config)
-        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
-        del old_prefs[library_id]
-
-    if library_config is None:
-        #print("get prefs from db")
-        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
-                                                 copy.deepcopy(default_prefs))
-    return library_config
-
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# fake out so I don't have to change the prefs calls anywhere.  The
-# Java programmer in me is offended by op-overloading, but it's very
-# tidy.
-class PrefsFacade():
-    def __init__(self,default_prefs):
-        self.default_prefs = default_prefs
-        self.libraryid = None
-        self.current_prefs = None
+    def _save_list(self,listcache):
+        #print("_save_list")
+        self.prefs['rejecturls'] = '\n'.join([x.to_line() for x in listcache.values()])
+        self.prefs.save_to_db()
+        self.listcache = None
         
-    def _get_prefs(self):
-        libraryid = get_library_uuid(get_gui().current_db)
-        if self.current_prefs == None or self.libraryid != libraryid:
-            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
-            self.libraryid = libraryid
-            self.current_prefs = get_library_config()
-        return self.current_prefs
+    def clear_cache(self):
+        self.listcache = None
+
+    # true if url is in list.
+    def check(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            return url in listcache
         
-    def __getitem__(self,k):            
-        prefs = self._get_prefs()
-        if k not in prefs:
-            # pulls from default_prefs.defaults automatically if not set
-            # in default_prefs
-            return self.default_prefs[k]
-        return prefs[k]
+    def get_note(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                return listcache[url].note
+            # not found
+            return ''
 
-    def __setitem__(self,k,v):
-        prefs = self._get_prefs()
-        prefs[k]=v
-        # self._save_prefs(prefs)
+    def get_full_note(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                return listcache[url].fullnote()
+            # not found
+            return ''
 
-    def __delitem__(self,k):
-        prefs = self._get_prefs()
-        if k in prefs:
-            del prefs[k]
+    def remove(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                del listcache[url]
+                self._save_list(listcache)
 
-    def save_to_db(self):
-        set_library_config(self._get_prefs())
-        
+    def add_text(self,rejecttext,addreasontext):
+        self.add(self._read_list_from_text(rejecttext,addreasontext).values())
+            
+    def add(self,rejectlist,clear=False):
+        with self.sync_lock:
+            if clear:
+                listcache=OrderedDict()
+            else:
+                listcache = self._get_listcache()
+            for l in rejectlist:
+                listcache[l.url]=l
+            self._save_list(listcache)
+
+    def get_list(self):
+        return self._get_listcache().values()
+            
+    def get_reject_reasons(self):
+        return self.prefs['rejectreasons'].splitlines()
+
+rejecturllist = RejectURLList(prefs)
 
-prefs = PrefsFacade(default_prefs)
-    
 class ConfigWidget(QWidget):
 
     def __init__(self, plugin_action):
@@ -195,12 +168,14 @@ class ConfigWidget(QWidget):
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
+        prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
 
         if self.readinglist_tab:
@@ -258,7 +233,7 @@ class ConfigWidget(QWidget):
         # error column
         prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
 
-        # cust cols
+        # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
             val = unicode(combo.itemData(combo.currentIndex()).toString())
@@ -272,6 +247,8 @@ class ConfigWidget(QWidget):
             colsnewonly[col] = checkbox.isChecked()
         prefs['custom_cols_newonly'] = colsnewonly
         
+        prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
+        
         prefs.save_to_db()
 
     def edit_shortcuts(self):
@@ -359,6 +336,13 @@ class BasicTab(QWidget):
 
         self.l.addSpacing(10)        
 
+        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
+        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.showmarked.setChecked(prefs['showmarked'])
+        self.l.addWidget(self.showmarked)
+
+        self.l.addSpacing(10)        
+
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
         self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
@@ -388,11 +372,37 @@ class BasicTab(QWidget):
         self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
         self.l.addWidget(self.lookforurlinhtml)
 
+        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
+        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
+        self.l.addWidget(self.checkforseriesurlid)
+
         self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
         self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.l.addSpacing(10)        
+
+        horz = QHBoxLayout()
+        
+        self.rejectlist = QPushButton('Edit Reject URL List', self)
+        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
+        self.rejectlist.clicked.connect(self.show_rejectlist)
+        horz.addWidget(self.rejectlist)
+        
+        self.reject_urls = QPushButton('Add Reject URLs', self)
+        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
+        self.reject_urls.clicked.connect(self.add_reject_urls)
+        horz.addWidget(self.reject_urls)
+        
+        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
+        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
+        self.reject_reasons.clicked.connect(self.show_reject_reasons)
+        horz.addWidget(self.reject_reasons)
+        
+        self.l.addLayout(horz)
+        
         self.l.insertStretch(-1)
         
     def set_collisions(self):
@@ -408,7 +418,45 @@ class BasicTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+
+    def show_rejectlist(self):
+        d = RejectListDialog(self,
+                             rejecturllist.get_list(),
+                             rejectreasons=rejecturllist.get_reject_reasons(),
+                             header="Edit Reject URLs List",
+                             show_delete=False,
+                             show_all_reasons=False)
+        d.exec_()
         
+        if d.result() != d.Accepted:
+            return
+        
+        rejecturllist.add(d.get_reject_list(),clear=True)
+        
+    def show_reject_reasons(self):
+        d = EditTextDialog(self,
+                           prefs['rejectreasons'],
+                           icon=self.windowIcon(),
+                           title="Reject Reasons",
+                           label="Customize Reject List Reasons",
+                           tooltip="Customize the Reasons presented when Rejecting URLs")
+        d.exec_()
+        if d.result() == d.Accepted:
+            prefs['rejectreasons'] = d.get_plain_text()
+                
+    def add_reject_urls(self):
+        d = EditTextDialog(self,
+                           "http://example.com/story.php?sid=5,Reason why I rejected it\nhttp://example.com/story.php?sid=6,Title by Author - Reason why I rejected it",
+                           icon=self.windowIcon(),
+                           title="Add Reject URLs",
+                           label="Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored.",
+                           tooltip="One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>",
+                           rejectreasons=rejecturllist.get_reject_reasons(),
+                           reasonslabel='Add this reason to all URLs added:')
+        d.exec_()
+        if d.result() == d.Accepted:
+            rejecturllist.add_text(d.get_plain_text(),d.get_reason_text())
+                
 class PersonalIniTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -437,7 +485,7 @@ class PersonalIniTab(QWidget):
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
 
-        self.defaults = QPushButton('View Defaults', self)
+        self.defaults = QPushButton('View Defaults (plugin-defaults.ini)', self)
         self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
@@ -448,7 +496,7 @@ class PersonalIniTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-
+        
 class ShowDefaultsIniDialog(QDialog):
 
     def __init__(self, icon, text, parent=None):
@@ -456,7 +504,7 @@ class ShowDefaultsIniDialog(QDialog):
         self.resize(600, 500)
         self.l = QVBoxLayout()
         self.setLayout(self.l)
-        self.label = QLabel("Plugin Defaults (Read-Only)")
+        self.label = QLabel("Plugin Defaults (plugin-defaults.ini) (Read-Only)")
         self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
         self.setWindowTitle(_('Plugin Defaults'))
         self.setWindowIcon(icon)
@@ -595,6 +643,8 @@ class GenerateCoverTab(QWidget):
             horz.addWidget(dropdown)
             self.sl.addLayout(horz)
         
+        self.sl.insertStretch(-1)
+        
         self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
         self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
         self.gcnewonly.setChecked(prefs['gcnewonly'])
@@ -605,8 +655,6 @@ class GenerateCoverTab(QWidget):
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             
-        self.l.insertStretch(-1)
-        
 class CountPagesTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -715,6 +763,7 @@ permitted_values = {
                      'language',
                      'series',
                      'characters',
+                     'ships',
                      'status',
                      'datePublished',
                      'dateUpdated',
@@ -732,10 +781,10 @@ permitted_values = {
                      'description',
                      'author',
                      'authorUrl',
-                     'formatname'
+                     'formatname',
+                     'version'
                      #,'formatext'   # not useful information.
                      #,'siteabbrev'
-                     #,'version'
                      ]
     }
 # no point copying the whole list.
@@ -751,9 +800,10 @@ titleLabels = {
     'status-I':'Status:In-Progress',
     'series':'Series',
     'characters':'Characters',
+    'ships':'Relationships',
     'datePublished':'Published',
     'dateUpdated':'Updated',
-    'dateCreated':'Packaged',
+    'dateCreated':'Created',
     'rating':'Rating',
     'warnings':'Warnings',
     'numChapters':'Chapters',
@@ -764,7 +814,7 @@ titleLabels = {
     'extratags':'Extra Tags',
     'title':'Title',
     'storyUrl':'Story URL',
-    'description':'Summary',
+    'description':'Description',
     'author':'Author',
     'authorUrl':'Author URL',
     'formatname':'File Format',
@@ -793,6 +843,15 @@ class CustomColumnsTab(QWidget):
         self.custcol_dropdowns = {}
         self.custcol_newonlycheck = {}
 
+        scrollable = QScrollArea()
+        scrollcontent = QWidget()
+        scrollable.setWidget(scrollcontent)
+        scrollable.setWidgetResizable(True)
+        self.l.addWidget(scrollable)
+
+        self.sl = QVBoxLayout()
+        scrollcontent.setLayout(self.sl)
+        
         for key, column in custom_columns.iteritems():
 
             if column['datatype'] in permitted_values:
@@ -823,15 +882,20 @@ class CustomColumnsTab(QWidget):
                     newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
                 horz.addWidget(newonlycheck)
                 
-                self.l.addLayout(horz)
+                self.sl.addLayout(horz)
         
-        self.l.insertStretch(-1)
+        self.sl.insertStretch(-1)
 
+        self.l.addSpacing(5)
+        self.allow_custcol_from_ini = QCheckBox('Allow custom_columns_settings from personal.ini to override',self)
+        self.allow_custcol_from_ini.setToolTip("The personal.ini parameter custom_columns_settings allows you to set custom columns to site specific values that aren't common to all sites.<br \>custom_columns_settings is ignored when this is off.")
+        self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
+        self.l.addWidget(self.allow_custcol_from_ini)
+        
         self.l.addSpacing(5)
         label = QLabel("Special column:")
         label.setWordWrap(True)
         self.l.addWidget(label)
-        self.l.addSpacing(5)
         
         horz = QHBoxLayout()
         label = QLabel("Update/Overwrite Error Column:")
@@ -847,8 +911,7 @@ class CustomColumnsTab(QWidget):
         self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
-        
-        
+
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
 
 
@@ -898,3 +961,4 @@ class StandardColumnsTab(QWidget):
             self.l.addLayout(horz)
         
         self.l.insertStretch(-1)
+

calibre-plugin/dialogs.py

@@ -7,32 +7,35 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback
+import traceback, re
+from functools import partial
+
+import urllib
+import email
 
 from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
-                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView )
+from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
+                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
+                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal)
 
-from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.gui2.complete2 import EditWithComplete
 
-from calibre import confirm_config_name
-from calibre.gui2 import dynamic
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
 
-SKIP='Skip'
-ADDNEW='Add New Book'
-UPDATE='Update EPUB if New Chapters'
-UPDATEALWAYS='Update EPUB Always'
-OVERWRITE='Overwrite if Newer'
-OVERWRITEALWAYS='Overwrite Always'
-CALIBREONLY='Update Calibre Metadata Only'
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
+
+SKIP=u'Skip'
+ADDNEW=u'Add New Book'
+UPDATE=u'Update EPUB if New Chapters'
+UPDATEALWAYS=u'Update EPUB Always'
+OVERWRITE=u'Overwrite if Newer'
+OVERWRITEALWAYS=u'Overwrite Always'
+CALIBREONLY=u'Update Calibre Metadata Only'
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
@@ -40,7 +43,79 @@ collision_order=[SKIP,
                  OVERWRITE,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
+
+anthology_collision_order=[UPDATE,
+                           UPDATEALWAYS,
+                           OVERWRITEALWAYS]
+
+class RejectUrlEntry:
+
+    matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
+    
+    def __init__(self,url_or_line,note=None,title=None,auth=None,
+                 addreasontext=None,fromline=False):
         
+        self.url=url_or_line
+        self.note=note
+        self.title=title
+        self.auth=auth
+        self.valid=False
+
+        if fromline:
+            mc = re.match(self.matchpat,url_or_line)
+            if mc:
+                #print("mc:%s"%mc.groupdict())
+                (url,title,auth,note) = mc.group('url','title','auth','note')
+                if not mc.group('title'):
+                    title=''
+                    auth=''
+                    note=mc.group('fullnote')
+                self.url=url
+                self.note=note
+                self.title=title
+                self.auth=auth
+        
+        if not self.note:
+            if addreasontext:
+                self.note = addreasontext
+            else:
+                self.note = ''
+        else:
+            if addreasontext:
+                self.note = self.note + ' - ' + addreasontext
+                
+        self.url = getNormalStoryURL(self.url)
+        self.valid = self.url != None
+                
+    def to_line(self):
+        # always 'url,'
+        return self.url+","+self.fullnote()
+        
+    def fullnote(self):
+        retval = ""
+        if self.title and self.auth:
+            retval = retval + "%s by %s"%(self.title,self.auth)
+            if self.note:
+                retval = retval + " - "
+                
+        if self.note:
+            retval = retval + self.note
+            
+        return retval
+
+# This is a more than slightly kludgey way to get
+# EditWithComplete to *not* alpha-order the reasons, but leave
+# them in the order entered.  If
+# calibre.gui2.complete2.CompleteModel.set_items ever changes,
+# this function will need to also.
+def complete_model_set_items_kludge(self, items):
+    items = [unicode(x.strip()) for x in items]
+    items = [x for x in items if x]
+    items = tuple(items)
+    self.all_items = self.current_items = items
+    self.current_prefix = ''
+    self.reset()
+    
 class NotGoingToDownload(Exception):
     def __init__(self,error,icon='dialog_error.png'):
         self.error=error
@@ -52,6 +127,36 @@ class NotGoingToDownload(Exception):
 class DroppableQTextEdit(QTextEdit):
     def __init__(self,parent):
         QTextEdit.__init__(self,parent)
+
+    def dropEvent(self,event):
+        # print("event:%s"%event)
+
+        mimetype='text/uri-list'
+
+        urllist=[]
+        filelist="%s"%event.mimeData().data(mimetype)
+        for f in filelist.splitlines():
+            #print("filename:%s"%f)
+            if f.endswith(".eml"):
+                fhandle = urllib.urlopen(f)
+                #print("file:\n%s\n\n"%fhandle.read())
+                msg = email.message_from_file(fhandle)
+                if msg.is_multipart():
+                    for part in msg.walk():
+                        #print("part type:%s"%part.get_content_type())
+                        if part.get_content_type() == "text/html":
+                            #print("URL list:%s"%get_urls_from_data(part.get_payload(decode=True)))
+                            urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
+                        if part.get_content_type() == "text/plain":
+                            #print("part content:text/plain")
+                            # print("part content:%s"%part.get_payload(decode=True))
+                            urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
+                else:
+                    urllist.extend(get_urls_from_text("%s"%msg))
+        
+        if urllist:
+            self.append("\n".join(urllist))
+        return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
         if source.hasUrls():
@@ -67,12 +172,11 @@ class DroppableQTextEdit(QTextEdit):
                             
 class AddNewDialog(SizePersistedDialog):
 
-    def __init__(self, gui, prefs, icon, url_list_text):
-        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
-        self.gui = gui
+    go_signal = pyqtSignal(object, object, object, object)
 
-        if prefs['adddialogstaysontop']:
-            QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
+    def __init__(self, gui, prefs, icon):
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
+        self.prefs = prefs
         
         self.setMinimumWidth(300)
         self.l = QVBoxLayout()
@@ -81,85 +185,190 @@ class AddNewDialog(SizePersistedDialog):
         self.setWindowTitle('FanFictionDownLoader')
         self.setWindowIcon(icon)
 
-        self.l.addWidget(QLabel('Story URL(s), one per line:'))
+        self.toplabel=QLabel("Toplabel")
+        self.l.addWidget(self.toplabel)
         self.url = DroppableQTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+        self.url.setToolTip("UrlTooltip")
         self.url.setLineWrapMode(QTextEdit.NoWrap)
-        self.url.setText(url_list_text)
         self.l.addWidget(self.url)
+
+        self.merge = self.newmerge = False
         
+        # elements to hide when doing merge.
+        self.mergehide = []
+        # elements to show again when doing *update* merge
+        self.mergeupdateshow = []
+
         horz = QHBoxLayout()
         label = QLabel('Output &Format:')
-        horz.addWidget(label)
+        self.mergehide.append(label)
+        
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
         self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
         self.fileform.activated.connect(self.set_collisions)
         
+        horz.addWidget(label)
         label.setBuddy(self.fileform)
         horz.addWidget(self.fileform)
         self.l.addLayout(horz)
+        self.mergehide.append(self.fileform)
 
         horz = QHBoxLayout()
-        label = QLabel('If Story Already Exists?')
-        horz.addWidget(label)
+        self.collisionlabel = QLabel("CollisionLabel")
+        horz.addWidget(self.collisionlabel)
         self.collision = QComboBox(self)
-        self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+        self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        label.setBuddy(self.collision)
+        self.collisionlabel.setBuddy(self.collision)
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
+        self.mergehide.append(self.collisionlabel)
+        self.mergehide.append(self.collision)
+        self.mergeupdateshow.append(self.collisionlabel)
+        self.mergeupdateshow.append(self.collision)
 
         horz = QHBoxLayout()
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
         self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
+        self.mergehide.append(self.updatemeta)
+        self.mergeupdateshow.append(self.updatemeta)
 
         self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
         self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
+        self.mergehide.append(self.updateepubcover)
         
         self.l.addLayout(horz)
 
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-        self.l.addWidget(button_box)
-        
-        if url_list_text:
-            button_box.button(QDialogButtonBox.Ok).setFocus()
+        self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        self.button_box.accepted.connect(self.ok_clicked)
+        self.button_box.rejected.connect(self.reject)
+        self.l.addWidget(self.button_box)
 
+    # invoke the 
+    def ok_clicked(self):
+        self.hide()
+        print("ok_clicked called")
+        self.go_signal.emit( self.get_ffdl_options(),
+                             self.get_urlstext(),
+                             self.merge,
+                             self.extrapayload )
+
+    def show_dialog(self,
+                    url_list_text,
+                    callback,
+                    show=True,
+                    merge=False,
+                    newmerge=True,
+                    extraoptions={},
+                    extrapayload=None):
+        # rather than mutex in ffdl_plugin, just bail here if it's
+        # already in use.
+        if self.isVisible(): return
+
+        try:
+            self.go_signal.disconnect()
+        except:
+            pass # if not already connected.
+        self.go_signal.connect(callback)
+
+        self.merge = merge
+        self.newmerge = newmerge
+        self.extraoptions = extraoptions
+        self.extrapayload = extrapayload
+        
+        if self.merge:
+            self.toplabel.setText('Story URL(s) for anthology, one per line:')
+            self.url.setToolTip('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.')
+            self.collisionlabel.setText('If Story Already Exists in Anthology?')
+            self.collision.setToolTip("What to do if there's already an existing story with the same URL in the anthology.")
+            for widget in self.mergehide:
+                widget.setVisible(False)
+            if not self.newmerge:
+                for widget in self.mergeupdateshow:
+                    widget.setVisible(True)
+        else:
+            for widget in self.mergehide:
+                widget.setVisible(True)
+            self.toplabel.setText('Story URL(s), one per line:')
+            self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+            self.collisionlabel.setText('If Story Already Exists?')
+            self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+
+        # Need to re-able after hiding/showing
+        self.setAcceptDrops(True)
+        self.url.setFocus()
+            
+        if self.prefs['adddialogstaysontop']:
+            QDialog.setWindowFlags ( self, Qt.Dialog | Qt.WindowStaysOnTopHint )
+        else:
+            QDialog.setWindowFlags ( self, Qt.Dialog )
+
+        if not self.merge:
+            self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
+
+        # add collision options
+        self.set_collisions()
+        
+        i = self.collision.findText(self.prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
+            
+        if not self.merge:
+            self.updateepubcover.setChecked(self.prefs['updateepubcover'])
+                
+        self.url.setText(url_list_text)
+        if url_list_text:
+            self.button_box.button(QDialogButtonBox.Ok).setFocus()
         # restore saved size.
         self.resize_dialog()
+        
+        if show: # so anthology update can be modal still.
+            self.show()
         #self.resize(self.sizeHint())
 
     def set_collisions(self):
         prev=self.collision.currentText()
         self.collision.clear()
-        for o in collision_order:
-            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+        if self.merge:
+            order = anthology_collision_order
+        else:
+            order = collision_order
+        for o in order:
+            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
                 self.collision.addItem(o)
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
         
     def get_ffdl_options(self):
-        return {
+        retval =  {
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
             'updateepubcover': self.updateepubcover.isChecked(),
-            }
+                }
+        
+        if self.merge:
+            retval['fileform']=='epub'
+            retval['updateepubcover']=True
+            if self.newmerge:
+                retval['updatemeta']=True
+                retval['collision']=ADDNEW
+            
+        return dict(retval.items() + self.extraoptions.items() )
 
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
@@ -176,10 +385,10 @@ class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title, url_text): 
+    def __init__(self, gui, title, url_text, epubmerge_plugin=None): 
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
-        self.gui = gui
         self.status=False
+        self.anthology=False
 
         self.setMinimumWidth(300)
         
@@ -187,28 +396,40 @@ class CollectURLDialog(SizePersistedDialog):
         self.setLayout(self.l)
 
         self.setWindowTitle(title)
-        self.l.addWidget(QLabel(title),0,0,1,2)
+        self.l.addWidget(QLabel(title),0,0,1,3)
         
         self.l.addWidget(QLabel("URL:"),1,0)
         self.url = QLineEdit(self)
         self.url.setText(url_text)
-        self.l.addWidget(self.url,1,1)
+        self.l.addWidget(self.url,1,1,1,2)
    
-        self.ok_button = QPushButton('OK', self)
-        self.ok_button.clicked.connect(self.ok)
-        self.l.addWidget(self.ok_button,2,0)
+        self.indiv_button = QPushButton('For Individual Books', self)
+        self.indiv_button.setToolTip('Get URLs and go to dialog for individual story downloads.')
+        self.indiv_button.clicked.connect(self.indiv)
+        self.l.addWidget(self.indiv_button,2,0)
+
+        self.merge_button = QPushButton('For Anthology Epub', self)
+        self.merge_button.setToolTip('Get URLs and go to dialog for Anthology download.\nRequires EpubMerge 1.3.1+ plugin.')
+        self.merge_button.clicked.connect(self.merge)
+        self.l.addWidget(self.merge_button,2,1)
+        self.merge_button.setEnabled(epubmerge_plugin!=None)
 
         self.cancel_button = QPushButton('Cancel', self)
         self.cancel_button.clicked.connect(self.cancel)
-        self.l.addWidget(self.cancel_button,2,1)
+        self.l.addWidget(self.cancel_button,2,2)
 
         # restore saved size.
         self.resize_dialog()
 
-    def ok(self):
+    def indiv(self):
         self.status=True
         self.accept()
 
+    def merge(self):
+        self.status=True
+        self.anthology=True
+        self.accept()
+
     def cancel(self):
         self.status=False
         self.reject()
@@ -219,7 +440,6 @@ class UserPassDialog(QDialog):
     '''
     def __init__(self, gui, site, exception=None):
         QDialog.__init__(self, gui)
-        self.gui = gui
         self.status=False
 
         self.l = QGridLayout()
@@ -278,7 +498,6 @@ class LoopProgressDialog(QProgressDialog):
                                  QString(), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
-        self.gui = gui
         self.book_list = book_list
         self.foreach_function = foreach_function
         self.finish_function = finish_function
@@ -327,7 +546,6 @@ class LoopProgressDialog(QProgressDialog):
 
     def do_when_finished(self):
         self.hide()
-        self.gui = None        
         # Queues a job to process these books in the background.
         self.finish_function(self.book_list)
 
@@ -376,7 +594,6 @@ class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
                  save_size_name='fanfictiondownloader_plugin:update list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.gui = gui
         
         self.setWindowTitle(header)
         self.setWindowIcon(icon)
@@ -394,11 +611,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         button_layout = QVBoxLayout()
         books_layout.addLayout(button_layout)
-        # self.move_up_button = QtGui.QToolButton(self)
-        # self.move_up_button.setToolTip('Move selected books up the list')
-        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
-        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
-        # button_layout.addWidget(self.move_up_button)
+
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
         self.remove_button = QtGui.QToolButton(self)
@@ -408,11 +621,6 @@ class UpdateExistingDialog(SizePersistedDialog):
         button_layout.addWidget(self.remove_button)
         spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem1)
-        # self.move_down_button = QtGui.QToolButton(self)
-        # self.move_down_button.setToolTip('Move selected books down the list')
-        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
-        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
-        # button_layout.addWidget(self.move_down_button)
 
         options_layout = QHBoxLayout()
 
@@ -438,7 +646,6 @@ class UpdateExistingDialog(SizePersistedDialog):
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
         label.setBuddy(self.collision)
         options_layout.addWidget(self.collision)
 
@@ -488,84 +695,6 @@ class UpdateExistingDialog(SizePersistedDialog):
             'updateepubcover': self.updateepubcover.isChecked(),
             }
 
-def display_story_list(gui, header, prefs, icon, books,
-                       label_text='',
-                       save_size_name='fanfictiondownloader_plugin:display list dialog',
-                       offer_skip=False):
-    all_good = True
-    for b in books:
-        if not b['good']:
-            all_good=False
-            break
-        
-    ## 
-    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
-        return True
-        pass
-            ## fake accept?
-    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
-                               label_text,
-                               save_size_name,
-                               offer_skip and all_good)
-    d.exec_()
-    return d.result() == d.Accepted
-
-class DisplayStoryListDialog(SizePersistedDialog):
-    def __init__(self, gui, header, prefs, icon, books,
-                 label_text='',
-                 save_size_name='fanfictiondownloader_plugin:display list dialog',
-                 offer_skip=False):
-        SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.name = save_size_name
-        self.gui = gui
-        
-        self.setWindowTitle(header)
-        self.setWindowIcon(icon)
-        
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-        title_layout = ImageTitleLayout(self, 'images/icon.png',
-                                        header)
-        layout.addLayout(title_layout)
-
-        self.books_table = StoryListTableWidget(self)
-        layout.addWidget(self.books_table)
-
-        options_layout = QHBoxLayout()
-        self.label = QLabel(label_text)
-        #self.label.setOpenExternalLinks(True)
-        #self.label.setWordWrap(True)
-        options_layout.addWidget(self.label)
-        
-        if offer_skip:
-            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
-            options_layout.addItem(spacerItem1)
-            self.again = QCheckBox('Show this again?',self)
-            self.again.setChecked(True)
-            self.again.stateChanged.connect(self.toggle)
-            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
-            options_layout.addWidget(self.again)
-        
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-
-        options_layout.addWidget(button_box)
-        
-        layout.addLayout(options_layout)
-        
-        # Cause our dialog size to be restored from prefs or created on first usage
-        self.resize_dialog()
-        self.books_table.populate_table(books)
-
-    def get_books(self):
-        return self.books_table.get_books()
-
-    def toggle(self, *args):
-        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
-
-    
-
 class StoryListTableWidget(QTableWidget):
 
     def __init__(self, parent):
@@ -625,11 +754,9 @@ class StoryListTableWidget(QTableWidget):
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
         
         url_cell = ReadOnlyTableWidgetItem(book['url'])
-        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
         self.setItem(row, 3, url_cell)
         
         comment_cell = ReadOnlyTableWidgetItem(book['comment'])
-        #comment_cell.setData(Qt.UserRole, QVariant(book))
         self.setItem(row, 4, comment_cell)
 
     def get_books(self):
@@ -663,52 +790,239 @@ class StoryListTableWidget(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def move_rows_up(self):
+class RejectListTableWidget(QTableWidget):
+
+    def __init__(self, parent,rejectreasons=[]):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+        self.rejectreasons = rejectreasons
+
+    def populate_table(self, reject_list):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(reject_list))
+        header_labels = ['URL', 'Title', 'Author', 'Note']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        # it's generally recommended to enable sort after pop, not
+        # before.  But then it needs to be sorted on a column and I'd
+        # rather keep the order given.
+        self.setSortingEnabled(True)
+        # row is just row number.
+        for row, rejectrow in enumerate(reject_list):
+            #print("populating table:%s"%rejectrow.to_line())
+            self.populate_table_row(row,rejectrow)
+
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(0, 100)
+        self.setMinimumColumnWidth(3, 100)
+        self.setMinimumSize(300, 0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, rej):
+
+        self.setItem(row, 0, ReadOnlyTableWidgetItem(rej.url))
+        self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
+        self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
+        
+        note_cell = EditWithComplete(self)
+        note_cell.lineEdit().mcompleter.model().set_items = \
+            partial(complete_model_set_items_kludge,
+                    note_cell.lineEdit().mcompleter.model())
+        
+        items = [rej.note]+self.rejectreasons
+        note_cell.update_items_cache(items)
+        note_cell.show_initial_value(rej.note)
+        note_cell.set_separator(None)
+        note_cell.setToolTip('Select or Edit Reject Note.')
+        self.setCellWidget(row, 3, note_cell)
+        
+    def remove_selected_rows(self):
         self.setFocus()
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        first_sel_row = rows[0].row()
-        if first_sel_row <= 0:
+        message = '<p>Are you sure you want to remove this URL from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+        if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
             return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in selrows:
-            self.swap_row_widgets(selrow - 1, selrow + 1)
-        scroll_to_row = first_sel_row - 1
-        if scroll_to_row > 0:
-            scroll_to_row = scroll_to_row - 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
 
-    def move_rows_down(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        last_sel_row = rows[-1].row()
-        if last_sel_row == self.rowCount() - 1:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in reversed(selrows):
-            self.swap_row_widgets(selrow + 2, selrow)
-        scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
 
-    def swap_row_widgets(self, src_row, dest_row):
-        self.blockSignals(True)
-        self.insertRow(dest_row)
-        for col in range(0, self.columnCount()):
-            self.setItem(dest_row, col, self.takeItem(src_row, col))
-        self.removeRow(src_row)
-        self.blockSignals(False)
+class RejectListDialog(SizePersistedDialog):
+    def __init__(self, gui, reject_list,
+                 rejectreasons=[],
+                 header="List of Books to Reject",
+                 icon='rotate-right.png',
+                 show_delete=True,
+                 show_all_reasons=True,
+                 save_size_name='ffdl:reject list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+      
+        self.setWindowTitle(header)
+        self.setWindowIcon(get_icon(icon))
+      
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, icon, header,
+                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+        layout.addLayout(title_layout)
+        rejects_layout = QHBoxLayout()
+        layout.addLayout(rejects_layout)
+
+        self.rejects_table = RejectListTableWidget(self,rejectreasons=rejectreasons)
+        rejects_layout.addWidget(self.rejects_table)
+
+        button_layout = QVBoxLayout()
+        rejects_layout.addLayout(button_layout)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+
+        if show_all_reasons:
+            self.reason_edit = EditWithComplete(self)
+            self.reason_edit.lineEdit().mcompleter.model().set_items = \
+                partial(complete_model_set_items_kludge,
+                        self.reason_edit.lineEdit().mcompleter.model())
+            
+            items = ['']+rejectreasons
+            self.reason_edit.update_items_cache(items)
+            self.reason_edit.show_initial_value('')
+            self.reason_edit.set_separator(None)
+            self.reason_edit.setToolTip("This will be added to whatever note you've set for each URL above.")
+            
+            horz = QHBoxLayout()
+            label = QLabel("Add this reason to all URLs added:")
+            label.setToolTip("This will be added to whatever note you've set for each URL above.")
+            horz.addWidget(label)
+            horz.addWidget(self.reason_edit)
+            horz.insertStretch(-1)
+            layout.addLayout(horz)
+                    
+        options_layout = QHBoxLayout()
+
+        if show_delete:
+            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
+            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks.setChecked(True)
+            options_layout.addWidget(self.deletebooks)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+      
+        layout.addLayout(options_layout)
+      
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.rejects_table.populate_table(reject_list)
+
+    def remove_from_list(self):
+        self.rejects_table.remove_selected_rows()
+
+    def get_reject_list(self):
+        rejectrows = []
+        for row in range(self.rejects_table.rowCount()):
+            url = unicode(self.rejects_table.item(row, 0).text()).strip()
+            title = unicode(self.rejects_table.item(row, 1).text()).strip()
+            auth = unicode(self.rejects_table.item(row, 2).text()).strip()
+            note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
+            rejectrows.append(RejectUrlEntry(url,note,title,auth,self.get_reason_text()))
+        return rejectrows
+
+    def get_reason_text(self):
+        try:
+            return unicode(self.reason_edit.currentText()).strip()
+        except:
+            # doesn't have self.reason_edit when editing existing list.
+            return None
+    
+    def get_deletebooks(self):
+        return self.deletebooks.isChecked()
+
+class EditTextDialog(QDialog):
+
+    def __init__(self, parent, text,
+                 icon=None, title=None, label=None, tooltip=None,
+                 rejectreasons=[],reasonslabel=None
+                 ):
+        QDialog.__init__(self, parent)
+        self.resize(600, 500)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel(label)
+        if title:
+            self.setWindowTitle(title)
+        if icon:
+            self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.textedit = QTextEdit(self)
+        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setText(text)
+        self.l.addWidget(self.textedit)
+
+        if tooltip:
+            self.label.setToolTip(tooltip)
+            self.textedit.setToolTip(tooltip)
+
+        if rejectreasons or reasonslabel:
+            self.reason_edit = EditWithComplete(self)
+
+            self.reason_edit.lineEdit().mcompleter.model().set_items = \
+                partial(complete_model_set_items_kludge,
+                        self.reason_edit.lineEdit().mcompleter.model())
+            
+            items = ['']+rejectreasons
+            self.reason_edit.update_items_cache(items)
+            self.reason_edit.show_initial_value('')
+            self.reason_edit.set_separator(None)
+            self.reason_edit.setToolTip(reasonslabel)
+            
+            if reasonslabel:
+                horz = QHBoxLayout()
+                label = QLabel(reasonslabel)
+                label.setToolTip(reasonslabel)
+                horz.addWidget(label)
+                horz.addWidget(self.reason_edit)
+                self.l.addLayout(horz)
+            else:
+                self.l.addWidget(self.reason_edit)
+            
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+
+    def get_plain_text(self):
+        return unicode(self.textedit.toPlainText())
+
+    def get_reason_text(self):
+        return unicode(self.reason_edit.currentText()).strip()
+    

calibre-plugin/ffdl_util.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+from StringIO import StringIO
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)
+
+def get_ffdl_personalini():
+    if prefs['includeimages']:
+        # this is a cheat to make it easier for users.
+        return '''[epub]
+include_images:true
+keep_summary_html:true
+make_firstimage_cover:true
+''' + prefs['personal.ini']
+    else:
+        return prefs['personal.ini']
+
+def get_ffdl_config(url,fileform="epub",personalini=None):
+    if not personalini:
+        personalini = get_ffdl_personalini()
+    site='unknown'
+    try:
+        site = adapters.getConfigSectionFor(url)
+    except Exception as e:
+        print("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
+    configuration = Configuration(site,fileform)
+    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+    configuration.readfp(StringIO(personalini))
+
+    return configuration
+
+def get_ffdl_adapter(url,fileform="epub",personalini=None):
+    return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
+    

calibre-plugin/jobs.py

@@ -10,18 +10,17 @@ __docformat__ = 'restructuredtext en'
 
 import time, os, traceback
 
-from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
-from calibre.utils.logging import Log
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
 
+from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config)
 # ------------------------------------------------------------------------------
 #
 #              Functions to perform downloads using worker jobs
@@ -40,11 +39,12 @@ def do_download_worker(book_list, options,
 
     print(options['version'])
     total = 0
+    alreadybad = []
     # Queue all the jobs
     print("Adding jobs for URLs:")
     for book in book_list:
+        print("%s"%book['url'])
         if book['good']:
-            print("%s"%book['url'])
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',
@@ -59,6 +59,9 @@ def do_download_worker(book_list, options,
             # job._modified_date = modified_date
             # job._existing_isbn = existing_isbn
             server.add_job(job)
+        else:
+            # was already bad before the subprocess ever started.
+            alreadybad.append(book)
     
     # This server is an arbitrary_n job, so there is a notifier available.
     # Set the % complete to a small number to avoid the 'unavailable' indicator
@@ -91,11 +94,11 @@ def do_download_worker(book_list, options,
             print("Successfully downloaded:")
             for book in book_list:
                 if book['good']:
-                    print(book['title'])
+                    print("%s %s"%(book['title'],book['url']))
             print("\nUnsuccessful:")
             for book in book_list:
                 if not book['good']:
-                    print(book['title'])
+                    print("%s %s"%(book['title'],book['url']))
             break
 
     server.close()
@@ -111,25 +114,33 @@ def do_download_for_worker(book,options):
     try:
         book['comment'] = 'Download started...'
 
-        ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        configuration = get_ffdl_config(book['url'],
+                                        options['fileform'],
+                                        options['personal.ini'])
         
         if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-            ffdlconfig.set("overrides","never_make_cover","true")
+            configuration.set("overrides","never_make_cover","true")
+
+        # images only for epub, even if the user mistakenly turned it
+        # on else where.
+        if options['fileform'] not in ("epub","html"):
+            configuration.set("overrides","include_images","false")
         
-        adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
+        adapter = adapters.getAdapter(configuration,book['url'])
         adapter.is_adult = book['is_adult'] 
         adapter.username = book['username'] 
         adapter.password = book['password']
+        adapter.setChaptersRange(book['begin'],book['end'])
         
         story = adapter.getStoryMetadataOnly()
         if 'calibre_series' in book:
-            # print("calibre_series:%s [%d]"%book['calibre_series'])
             adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
-        # else:
-        #     print("no calibre_series")
-        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+            
+        # set PI version instead of default.
+        if 'version' in options:
+            story.setMetadata('version',options['version'])
+            
+        writer = writers.getWriter(options['fileform'],configuration,adapter)
 
         outfile = book['outfile']
 
@@ -142,7 +153,21 @@ def do_download_for_worker(book,options):
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
 
-            adapter.setChaptersRange(book['begin'],book['end'])
+            # preserve logfile even on overwrite.
+            if 'epub_for_update' in book:
+                (urlignore,
+                 chaptercountignore,
+                 oldchaptersignore,
+                 oldimgsignore,
+                 oldcoverignore,
+                 calibrebookmarkignore,
+                 # only logfile set in adapter, so others aren't used.
+                 adapter.logfile) = get_update_data(book['epub_for_update'])
+
+                # change the existing entries id to notid so
+                # write_epub writes a whole new set to indicate overwrite.
+                if adapter.logfile:
+                    adapter.logfile = adapter.logfile.replace("span id","span notid")
             
             print("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
@@ -162,6 +187,17 @@ def do_download_for_worker(book,options):
              adapter.calibrebookmark,
              adapter.logfile) = get_update_data(book['epub_for_update'])
 
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if options['collision'] == UPDATE:
+                if chaptercount == urlchaptercount:
+                    book['comment']="Already contains %d chapters.  Reuse as is."%chaptercount
+                    book['outfile'] = book['epub_for_update'] # for anthology merge ops.
+                    return book
+
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if chaptercount > urlchaptercount:
+                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+                            
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)
 

calibre-plugin/prefs.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import copy
+
+from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import OVERWRITE
+from calibre_plugins.fanfictiondownloader_plugin.common_utils import get_library_uuid
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['showmarked'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['checkforseriesurlid'] = True
+default_prefs['injectseries'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+default_prefs['allow_custcol_from_ini'] = True
+
+default_prefs['std_cols_newonly'] = {}
+
+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
+def set_library_config(library_config,db):
+    db.prefs.set_namespaced(PREFS_NAMESPACE,
+                            PREFS_KEY_SETTINGS,
+                            library_config)
+
+def get_library_config(db):
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        #print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config,db)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        #print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def _get_db(self):
+        if self.passed_db:
+            return self.passed_db
+        else:
+            # In the GUI plugin we want current db so we detect when
+            # it's changed.  CLI plugin calls need to pass db in.
+            return get_gui().current_db
+    
+    def __init__(self,passed_db=None):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
+        self.passed_db=passed_db
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(self._get_db())
+        if self.current_prefs == None or self.libraryid != libraryid:
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config(self._get_db())
+        return self.current_prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        # self._save_prefs(prefs)
+
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        if k in prefs:
+            del prefs[k]
+
+    def save_to_db(self):
+        set_library_config(self._get_prefs(),self._get_db())
+        
+prefs = PrefsFacade()
+

downloader.py

@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-import logging
-
 import sys, os
 from os.path import normpath, expanduser, isfile, join
 from StringIO import StringIO
@@ -26,9 +24,25 @@ import string
 import ConfigParser
 from subprocess import call
 
-from fanficdownloader import adapters,writers,exceptions
-from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-from fanficdownloader.geturls import get_urls_from_page
+import logging
+if sys.version_info >= (2, 7):
+    # suppresses default logger.  Logging is setup in fanficdownload/__init__.py so it works in calibre, too.
+    rootlogger = logging.getLogger()
+    loghandler=logging.NullHandler()
+    loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
+    rootlogger.addHandler(loghandler)
+
+try:
+    from fanficdownloader import adapters,writers,exceptions
+    from fanficdownloader.configurable import Configuration
+    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficdownloader.geturls import get_urls_from_page
+except:
+    # running under calibre
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
@@ -41,15 +55,23 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     del writer
     return output_filename
 
-def main():
+def main(argv,
+         parser=None,
+         passed_defaultsini=None,
+         passed_personalini=None):
    # read in args, anything starting with -- will be treated as --<varible>=<value>
-   usage = "usage: %prog [options] storyurl"
-   parser = OptionParser(usage)
+   if not parser:
+       parser = OptionParser("usage: %prog [options] storyurl")
    parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), text or html", metavar="FORMAT")
+                     help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT")
+
+   if passed_defaultsini:
+       config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini"
+   else:
+       config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini"
    parser.add_option("-c", "--config",
                      action="append", dest="configfile", default=None,
-                     help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG")
+                     help=config_help, metavar="CONFIG")
    parser.add_option("-b", "--begin", dest="begin", default=None,
                      help="Begin with Chapter START", metavar="START")
    parser.add_option("-e", "--end", dest="end", default=None,
@@ -62,27 +84,28 @@ def main():
                      help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
+                     help="Update an existing epub with new chapters, give epub filename instead of storyurl.",)
    parser.add_option("--update-cover",
                      action="store_true", dest="updatecover",
                      help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
-                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
+                     help="Force overwrite of an existing epub, download and overwrite all chapters.",)
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
+   parser.add_option("-n", "--normalize-list",
+                     action="store_true", dest="normalize",default=False,
+                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug",
                      help="Show debug output while downloading.",)
    
-   (options, args) = parser.parse_args()
-
-   if options.debug:
-       logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
-   else:
-       logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+   (options, args) = parser.parse_args(argv)
 
+   if not options.debug:
+       logger = logging.getLogger("fanficdownloader")
+       logger.setLevel(logging.INFO)
    
    if len(args) != 1:
        parser.error("incorrect number of arguments")
@@ -90,16 +113,30 @@ def main():
    if options.update and options.format != 'epub':
        parser.error("-u/--update-epub only works with epub")
 
-   config = ConfigParser.SafeConfigParser()
+   ## Attempt to update an existing epub.
+   if options.update:
+       (url,chaptercount) = get_dcsource_chaptercount(args[0])
+       print "Updating %s, URL: %s" % (args[0],url)
+       output_filename = args[0]
+   else:
+       url = args[0]
+           
+   configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
+
+   if passed_defaultsini:
+       configuration.readfp(passed_defaultsini)
    
    if isfile(join(homepath,"defaults.ini")):
        conflist.append(join(homepath,"defaults.ini"))
    if isfile("defaults.ini"):
        conflist.append("defaults.ini")
        
+   if passed_personalini:
+       configuration.readfp(passed_personalini)
+   
    if isfile(join(homepath,"personal.ini")):
        conflist.append(join(homepath,"personal.ini"))
    if isfile("personal.ini"):
@@ -109,42 +146,42 @@ def main():
        conflist.extend(options.configfile)
       
    logging.debug('reading %s config file(s), if present'%conflist)
-   config.read(conflist)
+   configuration.read(conflist)
 
    try:
-       config.add_section("overrides")
+       configuration.add_section("overrides")
    except ConfigParser.DuplicateSectionError:
        pass
 
    if options.force:
-       config.set("overrides","always_overwrite","true")
+       configuration.set("overrides","always_overwrite","true")
 
+   if options.update:
+       configuration.set("overrides","output_filename",args[0])
+       
    if options.update and not options.updatecover:
-       config.set("overrides","never_make_cover","true")
+       configuration.set("overrides","never_make_cover","true")
+
+   # images only for epub, even if the user mistakenly turned it
+   # on else where.
+   if options.format not in ("epub","html"):
+       configuration.set("overrides","include_images","false")
        
    if options.options:
        for opt in options.options:
            (var,val) = opt.split('=')
-           config.set("overrides",var,val)
+           configuration.set("overrides",var,val)
 
-   if options.list:
-       retlist = get_urls_from_page(args[0], config)
+   if options.list or options.normalize:
+       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
        print "\n".join(retlist)
                
        return
 
    try:
-       ## Attempt to update an existing epub.
-       if options.update:
-           (url,chaptercount) = get_dcsource_chaptercount(args[0])
-           print "Updating %s, URL: %s" % (args[0],url)
-           output_filename = args[0]
-           config.set("overrides","output_filename",args[0])
-       else:
-           url = args[0]
-
-       adapter = adapters.getAdapter(config,url,options.format)
-
+       adapter = adapters.getAdapter(configuration,url)
+       adapter.setChaptersRange(options.begin,options.end)
+           
        ## Check for include_images and absence of PIL, give warning.
        if adapter.getConfig('include_images'):
            try:
@@ -158,7 +195,6 @@ def main():
                    print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
                    if not sys.stdin.readline().strip().lower().startswith('y'):
                        return
-               
 
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
@@ -201,16 +237,14 @@ def main():
                     adapter.calibrebookmark,
                     adapter.logfile) = get_update_data(args[0])
 
-                   writeStory(config,adapter,"epub")
+                   writeStory(configuration,adapter,"epub")
                    
        else:
            # regular download
            if options.metaonly:
                print adapter.getStoryMetadataOnly()
            
-           adapter.setChaptersRange(options.begin,options.end)
-           
-           output_filename=writeStory(config,adapter,options.format,options.metaonly)
+           output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        
        if not options.metaonly and adapter.getConfig("post_process_cmd"):
            metadata = adapter.story.metadata
@@ -230,5 +264,5 @@ def main():
 if __name__ == "__main__":
     #import time
     #start = time.time()
-    main()
+    main(sys.argv[1:])
     #print("Total time seconds:%f"%(time.time()-start))

example.ini

@@ -1,40 +1,103 @@
 ## This is an example of what your personal configuration might look
-## like.
+## like.  Uncomment options by removing the '#' in front of them.
 
 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
 #is_adult:true
 
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+#strip_chapter_numbers:true
+#add_chapter_numbers:true
+
+[epub]
+## include images from img tags in the body and summary of stories.
+## Images will be converted to jpg for size if possible.  Images work
+## in epub format only.  To get mobi or other format with images,
+## download as epub and use Calibre to convert.
+#include_images:true
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:true
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover:true
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+#image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+
 ## Most common, I expect will be using this to save username/passwords
-## for different sites.
+## for different sites.  Here are a few examples.  See defaults.ini
+## for the full list.
+
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
+## default is false
+#collect_series: true
 
 [www.ficwad.com]
 #username:YourUsername
 #password:YourPassword
 
+[www.twiwrite.net]
+#username:YourName
+#password:yourpassword
+## default is false
+#collect_series: true
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## The [defaults] section here will override the system [defaults],
-## but not format, site for site:format sections.
-[defaults]
-## Directories only useful in commandline or zip files.
-#output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
-#output_filename: books/${site}/${authorId}/${title}-${storyId}${formatext}
+[www.thewriterscoffeeshop.com]
+#username:YourName
+#password:yourpassword
+#is_adult:true
+## default is false
+#collect_series: true
+
+[www.fictionalley.org]
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+#is_adult:true
+
+[www.fimfiction.net]
+#is_adult:true
+#fail_on_password: false
+
+[www.tthfanfic.org]
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
 
-## For example, zip_output here will turn on zip for html and txt, but
-## not epub because the system [epub] section explicitly says
-## zip_output: false (epubs *are* specially formated zip files.)
-#zip_output: true
-#zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 
 ## This section will override anything in the system defaults or other
 ## sections here.
 [overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true

fanficdownloader/__init__.py

@@ -1 +1,16 @@
 # -*- coding: utf-8 -*-
+
+try:
+    # just a way to switch between web service and CLI/PI
+    import google.appengine.api 
+except:
+    import sys
+    if sys.version_info >= (2, 7):
+        import logging
+        logger = logging.getLogger(__name__)
+        loghandler=logging.StreamHandler()
+        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+        logger.addHandler(loghandler)
+        loghandler.setLevel(logging.DEBUG)
+        logger.setLevel(logging.DEBUG)
+

fanficdownloader/adapters/__init__.py

@@ -1,159 +1,205 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob, types
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-from .. import exceptions as exceptions
-
-## must import each adapter here.
-
-import adapter_test1
-import adapter_fanfictionnet
-import adapter_castlefansorg
-import adapter_fictionalleyorg
-import adapter_fictionpresscom
-import adapter_ficwadcom
-import adapter_fimfictionnet
-import adapter_harrypotterfanfictioncom
-import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
-import adapter_tenhawkpresentscom
-import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
-import adapter_tthfanficorg
-import adapter_twilightednet
-import adapter_twiwritenet
-import adapter_whoficcom
-import adapter_siyecouk
-import adapter_archiveofourownorg
-import adapter_ficbooknet
-import adapter_portkeyorg
-import adapter_mugglenetcom
-import adapter_hpfandomnet
-import adapter_thequidditchpitchorg
-import adapter_nfacommunitycom
-import adapter_midnightwhispersca
-import adapter_ksarchivecom
-import adapter_archiveskyehawkecom
-import adapter_squidgeorgpeja
-import adapter_libraryofmoriacom
-import adapter_wraithbaitcom
-import adapter_checkmatedcom
-import adapter_chaossycophanthexcom
-import adapter_dramioneorg
-import adapter_erosnsapphosycophanthexcom
-import adapter_lumossycophanthexcom
-import adapter_occlumencysycophanthexcom
-import adapter_phoenixsongnet
-import adapter_walkingtheplankorg
-import adapter_ashwindersycophanthexcom
-import adapter_thehexfilesnet
-import adapter_dokugacom
-import adapter_iketernalnet
-import adapter_onedirectionfanfictioncom
-import adapter_prisonbreakficnet
-import adapter_storiesofardacom
-import adapter_yourfanfictioncom
-import adapter_samdeanarchivenu
-import adapter_destinysgatewaycom
-import adapter_ncisfictioncom
-import adapter_stargateatlantisorg
-import adapter_thealphagatecom
-import adapter_fanfiktionde
-import adapter_ponyfictionarchivenet
-import adapter_sg1heliopoliscom
-import adapter_ncisficcom
-import adapter_nationallibrarynet
-import adapter_themasquenet
-import adapter_pretendercentrecom
-import adapter_darksolaceorg
-import adapter_finestoriescom
-import adapter_hpfanficarchivecom
-import adapter_svufictioncom
-import adapter_twilightarchivescom
-import adapter_wizardtalesnet
-import adapter_nhamagicalworldsus
-import adapter_hlfictionnet
-import adapter_grangerenchantedcom
-import adapter_dracoandginnycom
-import adapter_scarvesandcoffeenet
-import adapter_thepetulantpoetesscom
-import adapter_wolverineandroguecom
-import adapter_sinfuldesireorg
-
-## This bit of complexity allows adapters to be added by just adding
-## importing.  It eliminates the long if/else clauses we used to need
-## to pick out the adapter.
-    
-## List of registered site adapters.
-__class_list = []
-
-def imports():
-    for name, val in globals().items():
-        if isinstance(val, types.ModuleType):
-            yield val.__name__
-
-for x in imports():
-    if "fanficdownloader.adapters.adapter_" in x:
-        #print x
-        __class_list.append(sys.modules[x].getClass())
-            
-def getAdapter(config,url,fileform=None):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    logging.debug("trying url:"+url)
-    cls = getClassFor(domain)
-    if not cls and domain.startswith("www."):
-        domain = domain.replace("www.","")
-        logging.debug("trying site:without www: "+domain)
-        cls = getClassFor(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
-    if not cls:
-        logging.debug("trying site:www."+domain)
-        cls = getClassFor("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        adapter.setSectionOrder(adapter.getConfigSection(),fileform)
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getConfigSections():
-    return [cls.getConfigSection() for cls in __class_list]
-
-def getClassFor(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, re, sys, glob, types
+from os.path import dirname, basename, normpath
+import logging
+import urlparse as up
+
+logger = logging.getLogger(__name__)
+
+from .. import exceptions as exceptions
+from ..configurable import Configuration
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
+import adapter_archiveofourownorg
+import adapter_ficbooknet
+import adapter_portkeyorg
+import adapter_mugglenetcom
+import adapter_hpfandomnet
+import adapter_thequidditchpitchorg
+import adapter_nfacommunitycom
+import adapter_midnightwhispersca
+import adapter_ksarchivecom
+import adapter_archiveskyehawkecom
+import adapter_squidgeorgpeja
+import adapter_libraryofmoriacom
+import adapter_wraithbaitcom
+import adapter_checkmatedcom
+import adapter_chaossycophanthexcom
+import adapter_dramioneorg
+import adapter_erosnsapphosycophanthexcom
+import adapter_lumossycophanthexcom
+import adapter_occlumencysycophanthexcom
+import adapter_phoenixsongnet
+import adapter_walkingtheplankorg
+import adapter_ashwindersycophanthexcom
+import adapter_thehexfilesnet
+import adapter_dokugacom
+import adapter_iketernalnet
+import adapter_onedirectionfanfictioncom
+import adapter_prisonbreakficnet
+import adapter_storiesofardacom
+import adapter_samdeanarchivenu
+import adapter_destinysgatewaycom
+import adapter_ncisfictionnet
+import adapter_stargateatlantisorg
+import adapter_thealphagatecom
+import adapter_fanfiktionde
+import adapter_ponyfictionarchivenet
+import adapter_sg1heliopoliscom
+import adapter_ncisficcom
+import adapter_nationallibrarynet
+import adapter_themasquenet
+import adapter_pretendercentrecom
+import adapter_darksolaceorg
+import adapter_finestoriescom
+import adapter_hpfanficarchivecom
+import adapter_svufictioncom
+import adapter_twilightarchivescom
+import adapter_wizardtalesnet
+import adapter_nhamagicalworldsus
+import adapter_hlfictionnet
+import adapter_grangerenchantedcom
+import adapter_dracoandginnycom
+import adapter_scarvesandcoffeenet
+import adapter_thepetulantpoetesscom
+import adapter_wolverineandroguecom
+import adapter_sinfuldesireorg
+import adapter_merlinficdtwinscouk
+import adapter_thehookupzonenet
+import adapter_bloodtiesfancom
+import adapter_indeathnet
+import adapter_jlaunlimitedcom
+import adapter_qafficcom
+import adapter_efpfanficnet
+import adapter_potterficscom
+import adapter_efictionestelielde
+import adapter_dotmoonnet
+import adapter_pommedesangcom
+import adapter_restrictedsectionorg
+import adapter_imagineeficcom
+import adapter_buffynfaithnet
+import adapter_psychficcom
+import adapter_hennethannunnet
+
+## This bit of complexity allows adapters to be added by just adding
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
+    
+## List of registered site adapters.
+__class_list = []
+
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        __class_list.append(sys.modules[x].getClass())
+
+def getNormalStoryURL(url):
+    if not getNormalStoryURL.__dummyconfig:
+        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
+    # pulling up an adapter is pretty low over-head.  If
+    # it fails, it's a bad url.
+    try:
+        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
+        url = adapter.url
+        del adapter
+        return url
+    except:
+        return None;
+
+# kludgey function static/singleton
+getNormalStoryURL.__dummyconfig = None
+
+def getAdapter(config,url):
+
+    #logger.debug("trying url:"+url)
+    (cls,fixedurl) = getClassFor(url)
+    #logger.debug("fixedurl:"+fixedurl)
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getConfigSections():
+    return [cls.getConfigSection() for cls in __class_list]
+
+def getConfigSectionFor(url):
+    (cls,fixedurl) = getClassFor(url)
+    if cls:
+        return cls.getConfigSection()
+    
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getClassFor(url):
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+    
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    cls = getClassFromList(domain)
+    if not cls and domain.startswith("www."):
+        domain = domain.replace("www.","")
+        #logger.debug("trying site:without www: "+domain)
+        cls = getClassFromList(domain)
+        fixedurl = fixedurl.replace("http://www.","http://")
+    if not cls:
+        #logger.debug("trying site:www."+domain)
+        cls = getClassFromList("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+        
+    fixedurl = cls.stripURLParameters(fixedurl)
+    
+    return (cls,fixedurl)
+    
+def getClassFromList(domain):
+    for cls in __class_list:
+        if cls.matchesSite(domain):
+            return cls

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,12 +38,11 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Star Trek")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,7 +66,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -195,6 +195,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -205,7 +206,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -30,6 +31,8 @@ def getClass():
     return ArchiveOfOurOwnOrgAdapter
 
 
+logger = logging.getLogger(__name__)
+
 class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -48,13 +51,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
         else:
@@ -74,16 +77,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archiveofourown.org'
 
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.archiveofourown.org','archiveofourown.org']
-
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)(/chapters/\d+)?/?$"
+        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -108,14 +108,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         params['authenticity_token'] = data.split('input name="authenticity_token" type="hidden" value="')[1].split('" /></div>')[0]
 
         loginUrl = 'http://' + self.getSiteDomain() + '/user_sessions'
-        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['user_session[login]']))
     
         d = self._postUrl(loginUrl, params)
-        #logging.info(d)
+        #logger.info(d)
     
         if "Successfully logged in" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['user_session[login]']))
             raise exceptions.FailedToLogin(url,params['user_session[login]'])
             return False
@@ -132,8 +132,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
         metaurl = self.url+addurl
         url = self.url+'/navigate'+addurl
-        logging.info("url: "+url)
-        logging.info("metaurl: "+metaurl)
+        logger.info("url: "+url)
+        logger.info("metaurl: "+metaurl)
 
         try:
             data = self._fetchUrl(url)
@@ -168,7 +168,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
         if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
             self.story.setMetadata('author','Anonymous')
-            self.story.setMetadata('authorUrl',self.url)
+            self.story.setMetadata('authorUrl','http://archiveofourown.org/')
+            self.story.setMetadata('authorId','0')
         else:
             for a in alist:
                 self.story.addToList('authorId',a['href'].split('/')[2])
@@ -178,7 +179,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
         self.story.setMetadata('numChapters',len(chapters))
-        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
         for x in range(0,len(chapters)):
             # just in case there's tags, like <i> in chapter titles.
             chapter=chapters[x]
@@ -191,7 +192,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
         a = metasoup.find('blockquote',{'class':'userstuff'})
         if a != None:
-            self.setDescription(url,a.text)
+            self.setDescription(url,a)
             #self.story.setMetadata('description',a.text)
 		
         a = metasoup.find('dd',{'class':"rating tags"})
@@ -201,6 +202,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         a = metasoup.find('dd',{'class':"fandom tags"})
         fandoms = a.findAll('a',{'class':"tag"})
         for fandom in fandoms:
+            self.story.addToList('fandoms',fandom.string)
             self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
@@ -216,12 +218,15 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
+                self.story.addToList('freeformtags',genre.string)
                 self.story.addToList('genre',genre.string)
+                
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
                 if genre != "Gen":
+                    self.story.addToList('ao3categories',genre.string)
                     self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
@@ -229,13 +234,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chars = a.findAll('a',{'class':"tag"})
             for char in chars:
                 self.story.addToList('characters',char.string)
+                
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
-            chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
-                self.story.addToList('characters',char.string)
+            ships = a.findAll('a',{'class':"tag"})
+            for ship in ships:
+                self.story.addToList('ships',ship.string)
+		
+        a = metasoup.find('dd',{'class':"collections"})
+        if a != None:
+            collections = a.findAll('a')
+            for collection in collections:
+                self.story.addToList('collections',collection.string)
 		
-
         stats = metasoup.find('dl',{'class':'stats'})
         dt = stats.findAll('dt')
         dd = stats.findAll('dd')
@@ -246,6 +257,18 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             if 'Words:' in label:
                 self.story.setMetadata('numWords', value)
 				
+            if 'Comments:' in label:
+                self.story.setMetadata('comments', value)
+				
+            if 'Kudos:' in label:
+                self.story.setMetadata('kudos', value)
+				
+            if 'Hits:' in label:
+                self.story.setMetadata('hits', value)
+				
+            if 'Bookmarks:' in label:
+                self.story.setMetadata('bookmarks', value)
+				
             if 'Chapters:' in label:
                 if value.split('/')[0] == value.split('/')[1]:
                     self.story.setMetadata('status', 'Completed')
@@ -272,6 +295,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             series_url = 'http://'+self.host+'/fanfic/'+b['href']
             series_index = int(a.text.split(' ')[1])
             self.setSeries(series_name, series_index)
+            self.story.setMetadata('seriesUrl',series_url)
             
         except:
             # I find it hard to care if the series parsing fails
@@ -279,7 +303,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         chapter=bs.BeautifulSoup('<div class="story"></div>')
         data = self._fetchUrl(url)

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
@@ -65,20 +66,21 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archive.skyehawke.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['archive.skyehawke.com','www.skyehawke.com']
+
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
+        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
-
-
-        
+        return re.escape("http://")+r"(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -151,7 +153,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
                 rating.find('br').replaceWith('split')
                 rating=rating.text.replace("This story is rated",'').split('split')[0]
                 self.story.setMetadata('rating',rating)
-                logging.debug(self.story.getMetadata('rating'))
+                logger.debug(self.story.getMetadata('rating'))
 			
                 warnings=box.find('ol')
                 if warnings != None:
@@ -177,7 +179,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','asph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -99,13 +95,13 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Logout" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -117,7 +113,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -242,7 +238,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
 

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -0,0 +1,335 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BloodTiesFansComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','btf') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'bloodties-fans.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 4, 2 & 1.
+            # we'll try 1 first.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=561&warning=4
+        # viewstory.php?sid=561&warning=1
+        # viewstory.php?sid=561&warning=2
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fiction/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BuffyNFaithNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BuffyNFaithNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.setHeader()
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
+            nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+            #argh, this mangles the ampersands I need on metadata['storyUrl']
+            #will set it this way
+            self.story.setMetadata('storyUrl',nurl,condremoveentities=False)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','bnfnet')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'buffynfaith.net'
+
+    @classmethod
+    def stripURLParameters(cls,url):
+        "Only needs to be overriden if URL contains more than one parameter"
+        ## This adapter needs at least two parameters left on the URL, act and id
+        return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url)
+    
+    def setHeader(self):
+        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
+        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        
+    def getSiteExampleURLs(self):
+        return "http://buffynfaith.net/fanfictions/index.php?act=vie&id=963 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2"
+
+    def getSiteURLPattern(self):
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2
+        p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\
+            r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        dateformat = "%d %B %Y"
+        url = self.url
+        logger.debug("URL: "+url)
+        
+        #set a cookie to get past adult check
+        if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
+            cookie = cl.Cookie(version=0, name='my_age', value='yes',
+                               port=None, port_specified=False,
+                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
+                               path='/', path_specified=True,
+                               secure=False,
+                               expires=time.time()+10000,
+                               discard=False,
+                               comment=None,
+                               comment_url=None,
+                               rest={'HttpOnly': None},
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
+            self.setHeader()
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        if "ADULT CONTENT WARNING" in data: 
+            raise exceptions.AdultCheckRequired(self.url)
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        #stuff in <head>: description
+        svalue = soup.head.find('meta',attrs={'name':'description'})['content']
+        #self.story.setMetadata('description',svalue)
+        self.setDescription(url,svalue)
+        
+        #useful stuff in rest of doc, all contained in this:
+        doc = soup.body.find('div', id='my_wrapper')
+        
+        #first the site category (more of a genre to me, meh) and title, in this element:
+        mt = doc.find('div',attrs={'class':'maintitle'})
+        self.story.addToList('genre',mt.findAll('a')[1].string)
+        self.story.setMetadata('title',mt.findAll('a')[1].nextSibling[len('&nbsp;&raquo;&nbsp;'):])
+        del mt
+
+        #the actual category, for me, is 'Buffy: The Vampire Slayer'
+        #self.story.addToList('category','Buffy: The Vampire Slayer')
+        #No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini
+        
+        #then a block that sits in a table cell like so:
+        #(contains a lot of metadata)
+        mblock = doc.find('td', align='left', width = '70%').contents
+        while len(mblock) > 0:
+            i = mblock.pop(0)
+            if 'Author:' in i.string:
+                #drop empty space
+                mblock.pop(0)
+                #get author link
+                a = mblock.pop(0)
+                authre = re.escape('./index.php?act=bio&id=')+'(?P<authid>\d+)'
+                m = re.match(authre,a['href'])
+                self.story.setMetadata('author',a.string)
+                self.story.setMetadata('authorId',m.group('authid'))
+                authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('authorId'))
+                self.story.setMetadata('authorUrl',authurl,condremoveentities=False)
+                #drop empty space
+                mblock.pop(0)
+            if 'Rating:' in i.string:
+                self.story.setMetadata('rating',mblock.pop(0).strip())
+            if 'Published:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('datePublished',makeDate(date, dateformat))
+            if 'Last Updated:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('dateUpdated',makeDate(date, dateformat))
+            if 'Genre:' in i.string:
+                genres = mblock.pop(0).strip()
+                genres = genres.split('/')
+                for genre in genres: self.story.addToList('genre',genre)
+            #end ifs
+        #end while
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'ch' } )
+         
+        if select is None:
+           # no selector found, so it's a one-chapter story.
+           #self.chapterUrls.append((self.story.getMetadata('title'),url))
+           self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('storyId'),
+                                            o['value'])
+                title = u"%s" % o
+                title = stripHTML(title)
+                ts = title.split(' ',1)
+                title = ts[0]+'. '+ts[1]
+                self.chapterUrls.append((title,url))
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        ## Go scrape the rest of the metadata from the author's page.
+        data = self._fetchUrl(self.story.getMetadata('authorUrl'))
+        soup = bs.BeautifulSoup(data)
+        #find the story link and its parent div
+        storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
+        storydiv = storya.parent
+        #warnings come under a <spawn> tag. Never seen that before...
+        #appears to just be a line of freeform text, not necessarily a list
+        #optional
+        spawn = storydiv.find('spawn',{'id':'warnings'})
+        if spawn is not None:
+            warns = spawn.nextSibling.strip()
+            self.story.addToList('warnings',warns)
+        #some meta in spans - this should get all, even the ones jammed in a table
+        spans = storydiv.findAll('span')
+        for s in spans:
+            if s.string == 'Ship:':
+                list = s.nextSibling.strip().split()
+                self.story.extendList('ships',list)
+            if s.string == 'Characters:':
+                list = s.nextSibling.strip().split(',')
+                self.story.extendList('characters',list)
+            if s.string == 'Status:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('status',st)
+            if s.string == 'Words:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('numWords',st)
+
+        #reviews - is this worth having?
+        #ffnet adapter gathers it, don't know if anything else does
+        #or if it's ever going to be used!
+        a = storydiv.find('a',{'id':'bold-blue'})
+        if a:
+            revs = a.nextSibling.strip()[1:-1]
+            self.story.setMetadata('reviews',st)
+        else:
+            revs = '0'
+            self.story.setMetadata('reviews',st)
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'fanfiction'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        #remove all the unnecessary bookmark tags
+        [s.extract() for s in div('div',{'class':"tiny_box2"})]
+        
+        #is there a review link?
+        r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$"))
+        if r is not None:
+        #remove the review link and its parent div
+            r.parent.extract()
+        
+        #There might also be a link to the sequel on the last chapter
+        #I'm inclined to keep it in, but the URL needs to be changed from relative to absolute
+        #Shame there isn't proper series metadata available
+        #(I couldn't find it anyway)
+        s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$"))
+        if s is not None:
+            s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:]
+
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -71,7 +72,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -80,11 +81,6 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','cslf') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Castle") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y" # XXX
@@ -122,13 +118,13 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -150,7 +146,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -179,13 +175,14 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
-        
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
@@ -287,6 +284,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -297,7 +295,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','csph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        #self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -91,7 +87,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -216,6 +212,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -226,7 +223,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -44,25 +45,20 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         self.username = "" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-        
+
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
         self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
-        
+
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','chm')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y"
-            
+
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
@@ -82,7 +78,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             return True
         else:
             return False
-        
+
     def performLogin(self, url):
         params = {}
 
@@ -93,14 +89,14 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             params['name'] = self.getConfig("username")
             params['pass'] = self.getConfig("password")
         params['login'] = 'yes'
-        params['submit'] = 'login'	
-	
+        params['submit'] = 'login'
+
         loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
         d = self._fetchUrl(loginUrl,params)
         e = self._fetchUrl(url)
 
         if "Welcome back," not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['name']))
             raise exceptions.FailedToLogin(url,params['name'])
             return False
@@ -117,7 +113,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -135,33 +131,33 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # The actual text that is used to announce you need to be an
         # adult varies from site to site.  Again, print data before
         # the title search to troubleshoot.
-            
+
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
+
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
-        
+
         ## Title
         a = soup.findAll('span', {'class' : 'storytitle'})
         self.story.setMetadata('title',a[0].string)
-        
+
         # Find authorid and URL from... author url.
         a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
-			
+
         a = soup.find('select', {'name' : 'chapter'})
         if a == None:
             self.chapterUrls.append((self.story.getMetadata('title'),url))
         else:
             for chapter in a.findAll('option'):
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-			
+
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # eFiction sites don't help us out a lot with their meta data
@@ -174,7 +170,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             except:
                 return ""
 
-				
+
         # website does not keep track of word count, and there is no convenient way to calculate it
 
         summary = soup.find('fieldset')
@@ -182,21 +178,23 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         summary.name='div'
         self.setDescription(url,summary)
 
-		
+
         # <span class="label">Rated:</span> NC-17<br /> etc
         table = soup.findAll('div', {'class' : 'text'})[1]
         for labels in table.findAll('tr'):
             value = labels.findAll('td')[1]
             label = labels.findAll('td')[0]
 
-			
+
             if 'Rating' in stripHTML(label):
                 self.story.setMetadata('rating', stripHTML(value))
 
             if 'Ship' in stripHTML(label):
+                if value.string != "none/none":
+                    self.story.addToList('ships',value.string)
                 for char in value.string.split('/'):
                     if char != 'none':
-                        self.story.addToList('characters',char)		
+                        self.story.addToList('characters',char)
 
             if 'Status' in stripHTML(label):
                 if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
@@ -206,11 +204,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
 
             if 'Published' in stripHTML(label):
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
+
             if 'Updated' in stripHTML(label):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-			
-        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')	
+
+        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')
         for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
             a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
             if a != None:
@@ -220,20 +218,20 @@ class CheckmatedComAdapter(BaseSiteAdapter):
                     if 'genre' in stripHTML(label):
                         for genre in value.findAll('img'):
                             self.story.addToList('genre',genre['title'])
-        
-            
+
+
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
+
         div = soup.find('div', {'id' : 'resizeableText'})
         div.find('div', {'class' : 'storyTools'}).extract()
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','dksl')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Buffy the Vampire Slayer")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d %B %Y"
@@ -103,13 +99,13 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "User Account Page" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
+            logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
         else:
@@ -120,7 +116,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -256,6 +252,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -266,7 +263,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -86,7 +87,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -106,7 +107,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -217,6 +218,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -227,7 +229,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_dokugacom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,14 @@ class DokugaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        # www.dokuga.com has two 'sections', shown in URL as
+        # 'fanfiction' and 'spark' that change how things should be
+        # handled.
+        # http://www.dokuga.com/fanfiction/story/7528/1
+        # http://www.dokuga.com/spark/story/7299/1
+        self.section=self.parsedUrl.path.split('/',)[1]
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
@@ -56,14 +63,9 @@ class DokugaComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','dkg')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","InuYasha")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        if 'fanfiction' in self.story.getMetadata('section'):
+        if 'fanfiction' in self.section:
             self.dateformat = "%d %b %Y"
         else:
             self.dateformat = "%m-%d-%y"
@@ -79,6 +81,44 @@ class DokugaComAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'The author has disabled anonymous viewing for this story.' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url,soup):
+        params = {}
+
+        if self.password:
+            params['username'] = self.username
+            params['passwd'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['passwd'] = self.getConfig("password")
+        params['Submit'] = 'Submit'
+
+        # copy all hidden input tags to pick up appropriate tokens.
+        for tag in soup.findAll('input',{'type':'hidden'}):
+            params[tag['name']] = tag['value']
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['username']))
+    
+        d = self._postUrl(loginUrl, params)
+
+        if "Your session has expired. Please log in again." in d:
+            d = self._postUrl(loginUrl, params)
+
+        if "Logout" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['username']))
+            raise exceptions.FailedToLogin(url,params['username'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -86,7 +126,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -95,12 +135,18 @@ class DokugaComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+                
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url,soup)
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
@@ -122,17 +168,17 @@ class DokugaComAdapter(BaseSiteAdapter):
         # Find the chapters:
         chapters = soup.find('select').findAll('option')
         if len(chapters)==1:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/1'))
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1'))
         else:
             for chapter in chapters:
                 # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
         
         asoup = bs.BeautifulSoup(self._fetchUrl(alink))
         
-        if 'fanfiction' in self.story.getMetadata('section'):
+        if 'fanfiction' in self.section:
             asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
         
             #grab the rest of the metadata from the author's page
@@ -216,14 +262,10 @@ class DokugaComAdapter(BaseSiteAdapter):
             a=div.text.split('Words ')
             if len(a)==2: self.story.setMetadata('numWords', a[1])
             
-        
-
-
-            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DotMoonNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DotMoonNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL. www.dotmoon.net/library_view.php?storyid=3
+        self._setURL('http://' + self.getSiteDomain() + '/library_view.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dotm')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dotmoon.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/library_view.php?storyid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'You must be logged in to read adult-rated stories' in data \
+                or 'Password incorrect' in data \
+                or "That username does not exist" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['user'] = self.username
+            params['passwrd'] = self.password
+        else:
+            params['user'] = self.getConfig("username")
+            params['passwrd'] = self.getConfig("password")
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/board/index.php'
+        
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['user']))
+    
+        d = self._fetchUrl(loginUrl+'?action=login2&user='+params['user']+'&passwrd='+params['passwrd'])
+        d = self._fetchUrl(loginUrl)
+    
+        if "Show unread posts since last visit" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['user']))
+            raise exceptions.FailedToLogin(url,params['user'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Invalid story ID" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Invalid story ID.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        body=soup.findAll('body')[1]
+        body.find('table').extract()
+        
+        ## Title
+        a = body.find('b')
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
+        a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters: 'library_storyview.php?chapterid=3
+        chapters=body.findAll('a', href=re.compile(r"library_storyview.php\?chapterid=\d+$"))
+        if len(chapters)==0:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.")
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # other tags
+
+        labels = body.find('table', {'width':'390'}).findAll('td')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if label != None:
+                if 'Fandom' in label:
+                    self.story.addToList('category',value.string)
+                
+                if 'Setting' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Genre' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Style' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Rating' in label:
+                    self.story.addToList('rating',value.string)
+                
+                if 'Created' in label:
+                    self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+                if 'Updated' in label:
+                    self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+        
+                if 'Status' in label:
+                    if 'Completed' in value.string:
+                        self.story.setMetadata('status', 'Completed')
+                    else:
+                        self.story.setMetadata('status', 'In-Progress')
+                        
+        table=body.findAll('table', {'width':'400'})[1].find('td')
+        self.setDescription(url,stripHTML(table).split('Summary: ')[1])
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('blockquote')
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,13 +56,6 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','dcagn')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-        self.story.addToList("characters","Ginny Weasley")
-        self.story.addToList("characters","Draco Malfoy")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y"
@@ -99,13 +93,13 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -127,7 +121,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -152,7 +146,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -279,6 +273,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -289,12 +284,15 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'class' : 'listbox'})
+        
+        if None == div:
+            div = soup.find('div', {'id' : 'story'})
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','drmn')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d %B %Y"
@@ -97,13 +93,13 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -125,7 +121,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -165,6 +161,20 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
+        # Use banner as cover if found
+        coverurl = ''
+        img = soup.find('img',{'class':'banner'})
+        if img:
+            coverurl = img['src']
+            #print "Cover: "+coverurl
+        a = soup.find(text="This story has a banner; click to view.")
+        if a:
+            #print "A: "+ ', '.join("(%s, %s)" %tup for tup in a.parent.attrs)
+            coverurl = a.parent['href']
+            #print "Cover: "+coverurl
+        if coverurl:
+            self.setCoverImage(url,coverurl)
+
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
@@ -183,6 +193,22 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         for warning in warnings:
             self.story.addToList('warnings',warning.string)
 
+        themes=soup.findAll('a', {'class' : "tag-3"})
+        for theme in themes:
+            self.story.addToList('themes',theme.string)
+
+        hermiones=soup.findAll('a', {'class' : "tag-4"})
+        for hermione in hermiones:
+            self.story.addToList('hermiones',hermione.string)
+
+        dracos=soup.findAll('a', {'class' : "tag-5"})
+        for draco in dracos:
+            self.story.addToList('dracos',draco.string)
+
+        timelines=soup.findAll('a', {'class' : "tag-6"})
+        for timeline in timelines:
+            self.story.addToList('timeline',timeline.string)
+
         # utility method
         def defaultGetattr(d,k):
             try:
@@ -248,6 +274,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -258,7 +285,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -0,0 +1,222 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return EfictionEstelielDeAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class EfictionEstelielDeAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','eesd')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'efiction.esteliel.de'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        list = soup.find('div', {'class':'listbox'})
+        labelspan=list.find('span',{'class':'label'})
+        value = labelspan.nextSibling
+        label = labelspan.string
+        genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+        
+        labels = list.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while 'Rating' not in str(value):
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Words' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Category' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            if list.find('a', href=re.compile(r"series.php")) != None:
+                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
+                    # Find Series name from series URL.
+                    series_url = 'http://'+self.host+'/'+series['href']
+                    # use BeautifulSoup HTML parser to make everything easier to find.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+                    i=1
+                    for a in storyas:
+                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                            name=seriessoup.find('div', {'id' : 'pagetitle'})
+                            name.find('a').extract()
+                            self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
+                            i=0
+                            break
+                        i+=1
+                    if i == 0:
+                        break
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -0,0 +1,313 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return EFPFanFicNet
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class EFPFanFicNet(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','efp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.efpfanfic.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Fai il login e leggi la storia!' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Invia'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?sid='+self.story.getMetadata('storyId')
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if '<a class="menu" href="newaccount.php">' in d : # register for new account link
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'sid' } )
+    	 
+        if select is None:
+    	   # no selector found, so it's a one-chapter story.
+    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
+            for o in allOptions:
+                url = u'http://%s/%s' % ( self.getSiteDomain(),
+                                          o['value'])
+                # just in case there's tags, like <i> in chapter titles.
+                title = stripHTML(o)
+                self.chapterUrls.append((title,url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','Italian')
+        
+        # normalize story URL to first chapter if later chapter URL was given:
+        url = self.chapterUrls[0][1].replace('&i=1','')
+        logger.debug("Normalizing to URL: "+url)
+        self._setURL(url)
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        storya = None
+        authsoup = None
+        storyblock = None
+        authurl = self.story.getMetadata('authorUrl')
+        
+        ## author can have more than one page of stories.
+        while storyblock == None:
+
+            # no storya, but do have authsoup--we're looping on author pages.
+            if authsoup != None:
+                # last author link with offset should be the 'next' link.
+                authurl = u'http://%s/%s' % ( self.getSiteDomain(),
+                                              authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
+            
+            # Need author page for most of the metadata.
+            logger.debug("fetching author page: (%s)"%authurl)
+            authsoup = bs.BeautifulSoup(self._fetchUrl(authurl))
+            #print("authsoup:%s"%authsoup)
+
+            storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
+            for storya in storyas:
+                #print("======storya:%s"%storya)
+                storyblock = storya.findParent('div',{'class':'storybloc'})
+                #print("======storyblock:%s"%storyblock)
+                if storyblock != None:
+                    continue
+
+        self.setDescription(url,storyblock.find('div', {'class':'introbloc'}))
+
+        noteblock = storyblock.find('div', {'class':'notebloc'})
+        #print("%s"%noteblock)
+        notetext = ("%s" % noteblock).replace("<br />"," |")
+        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />␍
+        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />␍
+        # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>
+
+        cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        for item in notetext.split("|"):
+            if ":" in item:
+                (label,value) = item.split(":")
+                label=label.strip()
+                value=value.strip()
+            else:
+                label=value=item.strip()
+
+            if 'Pubblicata' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Aggiornata' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            if label == "Completa":
+                self.story.setMetadata('status', 'Completed')
+            
+            if label == "In corso":
+                self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Personaggi' in label:
+                for val in value.split(","):
+                    self.story.addToList('characters',val)
+                    
+            if 'Genere' in label:
+                for val in value.split(","):
+                    self.story.addToList('genre',val)
+                    
+            if 'Coppie' in label:
+                for val in value.split(","):
+                    self.story.addToList('ships',val)
+
+            if 'Avvertimenti' in label:
+                for val in value.split(","):
+                    if val != "None":
+                        self.story.addToList('warnings',val)
+
+            # 'extra' metadata for this adapter:
+                    
+            if 'Tipo di coppia' in label:
+                for val in value.split(","):
+                    self.story.addToList('type',val)
+                    
+            if 'Note' in label:
+                for val in value.split(","):
+                    if val != "None":
+                        self.story.addToList('notes',val)
+                    
+            if 'Contesto' in label:
+                self.story.setMetadata('context', value)
+
+            ## Note--efp doesn't provide word count.
+                
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?ssid=\d+&i=1"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'class' : 'storia'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        # remove any header and 'o:p' tags.
+        for tag in div.findAll("head") + div.findAll("o:p"):
+            tag.extract()
+
+        # change any html and body tags to div.
+        for tag in div.findAll("html") + div.findAll("body"):
+            tag.name='div'
+
+        # remove extra bogus doctype.
+        #<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+        return re.sub(r"<!DOCTYPE[^>]+>","",self.utf8FromSoup(url,div))

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','essph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y"
@@ -91,7 +87,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -111,7 +107,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -234,6 +230,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -244,7 +241,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 from urllib import unquote_plus
@@ -67,7 +68,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -75,12 +76,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # metadata and chapter list
 
         url = self.origurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
-            #print("\n===================\n%s\n===================\n"%data)
+            #logger.debug("\n===================\n%s\n===================\n"%data)
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -108,11 +109,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
                                            self.story.getMetadata('storyId'),
                                            chapcount+1)
-            print('=Trying newer chapter: %s' % tryurl)
+            logger.debug('=Trying newer chapter: %s' % tryurl)
             newdata = self._fetchUrl(tryurl)
             if "not found. Please check to see you are not using an outdated url." \
                     not in newdata:
-                print('=======Found newer chapter: %s' % tryurl)
+                logger.debug('=======Found newer chapter: %s' % tryurl)
                 soup = bs.BeautifulSoup(newdata)
         except:
             pass
@@ -157,10 +158,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if summarydiv:
             self.setDescription(url,stripHTML(summarydiv))
             
-        
-        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
+
+        graydiv = gui_table1i.find('div', {'class':'xgray'})
+        for b in graydiv.findAll('button'):
+            b.extract()
+        metatext = stripHTML(graydiv).replace('Hurt/Comfort','Hurt-Comfort')
+        #logger.debug("metatext:(%s)"%metatext)
         metalist = metatext.split(" - ")
-        #print("metatext:(%s)"%metalist)
+        #logger.debug("metalist:(%s)"%metalist)
 
         # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
 
@@ -176,9 +181,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
         goodgenres=True
         for g in genrelist:
-            print("g:(%s)"%g)
+            #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                print("g not in ffnetgenres")
+                logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -186,8 +191,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         donechars = False
         while len(metalist) > 0:
-            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Favs:') or metalist[0].startswith('Follows:'):
+            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
                 pass
+            elif  metalist[0].startswith('Reviews'):
+                self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Favs:'):
+                self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Follows:'):
+                self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
             elif  metalist[0].startswith('Updated'):
                 self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
             elif  metalist[0].startswith('Published'):
@@ -208,10 +219,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        if self.getConfig('include_images'):
+        # Try the larger image first.
+        try:
+            img = soup.find('img',{'class':'lazy cimage'})
+            self.setCoverImage(url,img['data-original'])
+        except:
             img = soup.find('img',{'class':'cimage'})
             if img:
-                self.story.addImgUrl(self,url,img['src'],self._fetchUrlRaw,cover=True)
+                self.setCoverImage(url,img['src'])
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapter' } )
@@ -235,7 +250,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
@@ -260,7 +275,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         div = soup.find('div', {'id' : 'storytextp'})
         
         if None == div:
-            logging.debug('div id=storytextp not found.  data:%s'%data)
+            logger.debug('div id=storytextp not found.  data:%s'%data)
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -25,6 +26,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
+
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
@@ -48,7 +50,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
@@ -94,12 +96,12 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         params['submit'] = 'Login...'
 
         loginUrl = 'https://ssl.fanfiktion.de/'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['nickname']))
         d = self._postUrl(loginUrl,params)
     
         if "Login erfolgreich" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['nickname']))
             raise exceptions.FailedToLogin(url,params['nickname'])
             return False
@@ -110,7 +112,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -139,7 +141,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        head = soup.find('div', {'style' : 'width:85%;float:left;'})
+        head = soup.find('div', {'class' : 'story-metadata-left-top'})
         a = head.find('a')
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
@@ -175,7 +177,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         self.setDescription(url,a['onmouseover'].split("', '")[1])
         
         td = tr[i].findAll('td')
-        self.story.addToList('category',td[1].string)
+        self.story.addToList('category',stripHTML(td[1]))
         self.story.setMetadata('rating', td[4].string)
         self.story.setMetadata('numWords', td[5].string)
  
@@ -183,7 +185,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffde has "floodlock" protection
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -18,6 +18,7 @@
 import time
 import datetime
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 from .. import translit
@@ -33,6 +34,8 @@ def getClass():
     return FicBookNetAdapter
 
 
+logger = logging.getLogger(__name__)
+
 class FicBookNetAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -49,7 +52,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
@@ -75,7 +78,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
         url=self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
         try:
             data = self._fetchUrl(url)
         except urllib2.HTTPError, e:
@@ -95,14 +98,14 @@ class FicBookNetAdapter(BaseSiteAdapter):
         ## Title
         a = soup.find('h1')
         self.story.setMetadata('title',stripHTML(a))
-        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
+        logger.debug("Title: (%s)"%self.story.getMetadata('title'))
         
         # Find authorid and URL from... author url.
         a = table.find('a')
         self.story.setMetadata('authorId',a.text) # Author's name is unique
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.text)
-        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
+        logger.debug("Author: (%s)"%self.story.getMetadata('author'))
 
         # Find the chapters:
         chapters = soup.find('div', {'class' : 'part_list'})
@@ -123,7 +126,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
             pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
             update=pubdate
 
-        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
 
         if not ',' in pubdate:
             pubdate=datetime.date.today().strftime(self.dateformat)
@@ -201,13 +204,13 @@ class FicBookNetAdapter(BaseSiteAdapter):
                 break
 				
         summary=soup.find('span', {'class' : 'urlize'})
-        self.setDescription(url,summary.text)
+        self.setDescription(url,summary)
         #self.story.setMetadata('description', summary.text)
             
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,7 +38,6 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query correct
@@ -45,7 +45,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if m:
             self.story.setMetadata('authorId',m.group('auth'))
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL(url)
         else:
@@ -69,7 +69,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if self.is_adult or self.getConfig("is_adult"):
             params={'iamold':'Yes',
                     'action':'ageanswer'}
-            logging.info("Attempting to get cookie for %s" % url)
+            logger.info("Attempting to get cookie for %s" % url)
             ## posting on list doesn't work, but doesn't hurt, either.
             data = self._postUrl(url,params)
         else:
@@ -80,7 +80,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
         ## could be either chapter list page or one-shot text page.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._postFetchWithIAmOld(url)
@@ -108,7 +108,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             storya = soup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
             self._setURL(storya['href'])
             url=self.url
-            logging.debug("Normalizing to URL: "+url)
+            logger.debug("Normalizing to URL: "+url)
             ## title's right there...
             self.story.setMetadata('title',storya.string)
             data = self._fetchUrl(url)
@@ -194,7 +194,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
 	# find <!-- headerend --> & <!-- footerstart --> and
@@ -223,7 +223,16 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             
         if not data or not text:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
+        # not sure how, but we can get html, etc tags still in some
+        # stories.  That breaks later updates because it confuses
+        # epubutils.py
+        for tag in text.findAll('head'):
+            tag.extract()
+        
+        for tag in text.findAll('body') + text.findAll('html'):
+            tag.name = 'div'
+            
         return self.utf8FromSoup(url,text)
 
 def getClass():

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time
@@ -42,10 +43,10 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     @staticmethod
     def getSiteDomain():
-        return 'www.ficwad.com'
+        return 'ficwad.com'
 
     def getSiteExampleURLs(self):
-        return "http://www.ficwad.com/story/137169"
+        return "http://ficwad.com/story/137169"
 
     def getSiteURLPattern(self):
         return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"
@@ -61,12 +62,12 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             params['password'] = self.getConfig("password")
 
         loginUrl = 'http://' + self.getSiteDomain() + '/account/login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['username']))
         d = self._postUrl(loginUrl,params)
 
         if "Login attempt failed..." in d:
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['username']))
             raise exceptions.FailedToLogin(url,params['username'])
             return False
@@ -79,7 +80,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         # metadata and chapter list
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
@@ -96,7 +97,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             # normalize story URL on chapter list.
             self.story.setMetadata('storyId',storya['href'].split('/',)[2])
             url = "http://"+self.getSiteDomain()+storya['href']
-            logging.debug("Normalizing to URL: "+url)
+            logger.debug("Normalizing to URL: "+url)
             self._setURL(url)
             try:
                 soup = bs.BeautifulSoup(self._fetchUrl(url))
@@ -124,7 +125,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # description
         storydiv = soup.find("div",{"id":"story"})
-        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p.string)
+        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p)
         #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
 
         # most of the meta data is here:
@@ -201,7 +202,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
@@ -78,8 +79,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
             apiData = json.loads(apiResponse)
             
-            # Unfortunately, we still need to load the story index page to parse the characters
+            # Unfortunately, we still need to load the story index
+            # page to parse the characters.  And chapters, now, too.
             data = self._fetchUrl(self.url)
+            soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -92,9 +95,6 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "/images/missing_story.png" in data:
             raise exceptions.StoryDoesNotExist(self.url)
         
-        if "Invalid story id" in apiData.values():
-            raise exceptions.StoryDoesNotExist(self.url)
-        
         if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
@@ -109,43 +109,81 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             else:
                 raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
          
+        if "Invalid story id" in apiData.values():
+            raise exceptions.StoryDoesNotExist(self.url)
+        
         storyMetadata = apiData["story"]    
             
-        self.story.setMetadata("title", storyMetadata["title"])
+        ## Title
+        a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',a.string)
+        
+        # self.story.setMetadata("title", storyMetadata["title"])
+        # if not storyMetadata["title"]:
+        #     raise exceptions.FailedToDownload("%s doesn't have a title in the API.  This is a known fimfiction.net bug with titles containing ."%self.url)
+        
         self.story.setMetadata("author", storyMetadata["author"]["name"])
         self.story.setMetadata("authorId", storyMetadata["author"]["id"])
         self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
+
+        # chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
         
-        chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
-        for chapter in chapters:
-            self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
-        self.story.setMetadata("numChapters", len(self.chapterUrls))
+        # ## this is bit of a kludge based on the assumption all the
+        # ## 'bad' chapters will be at the end.
+        # ## limit down to the number of chapters reported by chapter_count.
+        # chapters = chapters[:storyMetadata["chapter_count"]] 
         
+        # for chapter in chapters:
+        #     self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
+        # self.story.setMetadata("numChapters", len(self.chapterUrls))
+
+        for chapter in soup.findAll('a',{'class':'chapter_link'}):
+            self.chapterUrls.append((stripHTML(chapter), 'http://'+self.host+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
         # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
-        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
+        # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
         # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
         # status to leave unchanged.
-        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
+        # Nov2012 - 'On Hiatus' is now passed, too.  It's easy now for users to change/remove if they want
+        # with replace_metadata
+        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata("status", status)
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
-            
+
+        ## Warnings aren't included in the API.
+        bottomli = soup.find('li',{'class':'bottom'})
+        if bottomli:
+            bottomspans = bottomli.findAll('span')
+            # the first span in bottom is the rating, obtained above.
+            if bottomspans and len(bottomspans) > 1:
+                for warning in bottomspans[1:]:
+                    self.story.addToList('warnings',warning.string)
+                
+        
         for category in storyMetadata["categories"]:
             if storyMetadata["categories"][category]:
                 self.story.addToList("genre", category) 
 
-        self.story.addToList("category", "My Little Pony")
-        
         self.story.setMetadata("numWords", str(storyMetadata["words"]))
         
-        
         # fimfic is the first site with an explicit cover image.
-        if self.getConfig('include_images') and "image" in storyMetadata.keys():
-            coverurl = storyMetadata["image"]
-            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
+        if "image" in storyMetadata.keys():
+            if "full_image" in storyMetadata:
+                coverurl = storyMetadata["full_image"]
+            else:
+                coverurl = storyMetadata["image"]
+            if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
-            self.story.addImgUrl(self,self.url,coverurl,self._fetchUrlRaw,cover=True)
-            
-        self.setDescription(self.url, storyMetadata["description"])
+
+            self.setCoverImage(self.url,coverurl)
+
+        # fimf has started including extra stuff inside the description div.
+        descdivstr = "%s"%soup.find("div", {"class":"description"})
+        hrstr="<hr />"
+        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        self.setDescription(self.url,descdivstr)
         
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
@@ -153,15 +191,34 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
         rawDateUpdated = storyMetadata["date_modified"]
         self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
-        
-        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
-        for character in [character_icon["title"] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
-            self.story.addToList("characters", character)
+
+        chars = soup.find("div", {"class":"inner_data"})
+        # fimfic stopped putting the char name on or around the char
+        # icon now for some reason.  Pull it from the image name with
+        # some heuristics.
+        for character in [character_icon["src"] for character_icon in chars.findAll("img", {"class":"character_icon"})]:
+            # //static.fimfiction.net/images/characters/twilight_sparkle.png
+            # 5th split /, remove last four, replace _, capitolize every word(title())
+            char = character.split('/')[5][:-4].replace('_',' ').title()
+            if char == 'Oc':
+                char = "OC"
+            if char == 'Cmc':
+                char = "Cutie Mark Crusaders"
+            self.story.addToList("characters", char)
             
+        # extra site specific metadata
+        extralist = ["likes","dislikes","views","total_views","short_description"]
+        for metakey in extralist:
+            if metakey in storyMetadata:
+                value = storyMetadata[metakey]
+                if not isinstance(value,basestring):
+                    value = unicode(value)
+                self.story.setMetadata(metakey, value)
+        
             
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
+        logger.debug('Getting chapter text from: %s' % url)
+        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -49,7 +50,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
@@ -95,13 +96,13 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         params['submit'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['theusername']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "My Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['theusername']))
             raise exceptions.FailedToLogin(url,params['theusername'])
             return False
@@ -114,7 +115,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -186,6 +187,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
             i = a.parent.text.split('(')[1].split(')')[0]
             self.setSeries(a.text, i)
+            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
         except:
             pass
         try:
@@ -232,7 +234,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,8 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.section=self.parsedUrl.path.split('/',)[1]
              
         # normalized story URL.
         if "malfoymanor" in self.parsedUrl.netloc:
@@ -60,11 +61,6 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','gech')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%b/%Y"
@@ -105,17 +101,17 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         params['cookiecheck'] = '1'
         params['submit'] = 'Submit'
     
-        if "enchant" in self.story.getMetadata('section'):
+        if "enchant" in self.section:
             loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
         else:
             loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -137,7 +133,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -162,7 +158,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -196,7 +192,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -267,7 +263,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -279,6 +275,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -289,10 +286,10 @@ class GrangerEnchantedCom(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'id' : 'story1'})
 

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,12 +38,11 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only psid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
@@ -73,7 +73,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -182,7 +182,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         ## most adapters use BeautifulStoneSoup here, but non-Stone
         ## allows nested div tags.

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HennethAnnunNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HennethAnnunNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','htan')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.henneth-annun.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "We're sorry. This story is not available." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: This story is not available.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h2', {'id':'page_heading'})
+        self.story.setMetadata('title',a.string)
+
+        # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
+        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        self.story.setMetadata('numWords', soup.find('tr', {'class':'foot'}).findAll('td')[1].text)
+
+        self.setDescription(url,soup.find('div', {'id':'summary'}))
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        info = soup.find('div', {'id':'storyinformation'})
+        labels=info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+                
+            if 'Completion' in label:
+                if 'Complete' in value.string:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Era:' in label:
+                self.story.addToList('category',value.string)
+
+            if 'Genre' in label:
+                self.story.addToList('genre',value.string)
+                
+        labels=info.findAll('strong')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if 'Author' in label:
+                value=value.nextSibling
+                self.story.setMetadata('authorId',value['href'].split('=')[1])
+                self.story.setMetadata('authorUrl','http://'+self.host+'/'+value['href'])
+                self.story.setMetadata('author',value.string)
+                
+            if 'Post' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated:' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+        for char in soup.findAll('a', href=re.compile(r"/resources/bios_view.cfm\?scid=\d+")):
+            self.story.addToList('characters',stripHTML(char))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'block chapter'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','hlf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Highlander")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y"
@@ -81,7 +77,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -210,6 +206,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -220,7 +217,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','hpfdm') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d" # XXX
@@ -84,7 +80,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -185,21 +181,22 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
             value = td.nextSibling.string
             #print("\nlabel:%s\nvalue:%s\n"%(label,value))
 
-            if 'Category' in label:
+            if 'Category' in label and value:
                 cats = td.parent.findAll('a',href=re.compile(r'categories.php'))
                 catstext = [cat.string for cat in cats]
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
-            if 'Characters' in label:
+            if 'Characters' in label and value: # this site can have Character label with no
+                                                # values, apparently.  Others as a precaution.
                 for char in value.split(','):
                     self.story.addToList('characters',char.strip())
 
-            if 'Genre' in label:
+            if 'Genre' in label and value:
                 for genre in value.split(','):
                     self.story.addToList('genre',genre.strip())
 
-            if 'Warnings' in label:
+            if 'Warnings' in label and value:
                 for warning in value.split(','):
                     if warning.strip() != 'none':
                         self.story.addToList('warnings',warning.strip())
@@ -213,7 +210,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # There's no good wrapper around the chapter text. :-/

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','hpffa')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"
@@ -81,7 +77,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -197,6 +193,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -207,7 +204,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ike')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","InuYasha")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"
@@ -97,13 +93,13 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -125,7 +121,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -163,7 +159,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -272,7 +268,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -0,0 +1,288 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ImagineEFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ImagineEFicComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ime')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'imagine.e-fic.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_indeathnet.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return InDeathNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class InDeathNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+		
+		  # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/')
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','idn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'indeath.net'
+
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/"
+
+    def getSiteURLPattern(self):
+        # http://www.indeath.net/blog/archive/169-ransom-in-death/
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"/blog/(archive/)?(?P<id>\d+)\-(?P<name>[a-z0-9\-]*)/?$"     
+        
+            
+    def getDateFromComponents(self, postmonth, postday):
+        ym = re.search("Entries\ in\ (?P<mon>January|February|March|April|May|June|July|August|September|October|November|December)\ (?P<year>\d{4})",postmonth)
+        d = re.search("(?P<day>\d{2})\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)",postday)
+        postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
+        return postdate
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        try:
+            data = self._fetchUrl(url)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+                
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        h = soup.find('a', id="blog_title")
+        t = h.find('span')
+        self.story.setMetadata('title',t.contents[0].string.strip())
+		
+        s = t.find('div')        
+        if s != None:
+            self.setDescription(url,s)
+		
+        # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required)
+        a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$"))		#http://www.indeath.net/user/9083-cyrex/
+        m = re.search('http://www.indeath.net/user/(?P<id>\d+)\-(?P<name>[a-z0-9]*)/$',a['href'])
+        self.story.setMetadata('authorId',m.group('id'))
+        self.story.setMetadata('authorUrl',a['href'])
+        self.story.setMetadata('author',m.group('name'))
+
+
+
+
+        # Find the chapters:
+        chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
+
+        #reverse the list since newest at the top
+        chapters.reverse()
+        
+        # Get date published & updated from first & last entries
+        posttable=soup.find('div', id="main_column")
+        
+        postmonths=posttable.findAll('th', text=re.compile(r'Entries\ in\ '))
+        postmonths.reverse()
+        
+        postdates=posttable.findAll('span', _class="desc", text=re.compile('\d{2}\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'))
+        postdates.reverse()
+        
+        self.story.setMetadata('datePublished',self.getDateFromComponents(postmonths[0],postdates[0]))
+        self.story.setMetadata('dateUpdated',self.getDateFromComponents(postmonths[len(postmonths)-1],postdates[len(postdates)-1]))
+        
+        # Process List of Chapters              
+        self.story.setMetadata('numChapters',len(chapters))
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        for x in range(0,len(chapters)):
+            # just in case there's tags, like <i> in chapter titles.
+            chapter=chapters[x]
+            if len(chapters)==1:
+                self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
+            else:
+            	ct = stripHTML(chapter)
+            	tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+            	if tnew:
+            		chaptertitle = tnew.group('newtitle')
+            	else:
+            		chaptertitle = ct
+                self.chapterUrls.append((chaptertitle,chapter['href']))
+	
+		
+        
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+		
+        #chapter=bs.BeautifulSoup('<div class="story"></div>')
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))
+
+        chapter = soup.find("div", "entry_content")
+        
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chapter)
+

fanficdownloader/adapters/adapter_jlaunlimitedcom.py

@@ -0,0 +1,260 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return JLAUnlimitedComAdapter
+
+
+class JLAUnlimitedComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        self._setURL('http://' + self.getSiteDomain() + '/eFiction1.1/viewstory.php?sid='+self.story.getMetadata('storyId'))
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','jla')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.jlaunlimited.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=")+r"\d+$"
+
+
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # Assume that if there is a url with 'warning=#' in the page then it is a 'check' page
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+                
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eFiction1.1/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        div = soup.find('div', {'id' : 'story'})
+        
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,div)
+

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ksa') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Star Trek") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b/%d/%Y" # XXX
@@ -104,7 +100,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -137,7 +133,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -289,6 +285,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -299,7 +296,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -56,11 +57,6 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','lom')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Lord of the Rings")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"
@@ -89,7 +85,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -109,7 +105,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -229,6 +225,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -239,7 +236,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','lsph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -91,7 +87,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -216,6 +212,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -226,7 +223,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -42,7 +43,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
         else:
@@ -66,7 +67,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -188,7 +189,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data=self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,
@@ -214,7 +215,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
             return self.utf8FromSoup(url,anchor)
         
         else:
-            logging.debug('Using kludgey text find for older mediaminer story.')
+            logger.debug('Using kludgey text find for older mediaminer story.')
             ## Some older mediaminer stories are unparsable with BeautifulSoup.
             ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
             ## Story stuff falls between:

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -27,22 +28,15 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return YourFanfictionComAdapter
+    return MerlinFicDtwinsCoUk
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class YourFanfictionComAdapter(BaseSiteAdapter):
+class MerlinFicDtwinsCoUk(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # yourfanfiction.com blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted as us and offered to 'whitelist our IP'.  Clearly,
-        # that wouldn't work, but it does let me do this in good
-        # conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
-
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -54,22 +48,22 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','yff')
+        self.story.setMetadata('siteabbrev','mrfd')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
+        self.dateformat = "%b %d, %Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.yourfanfiction.com'
+        return 'merlinfic.dtwins.co.uk'
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
@@ -77,6 +71,41 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -92,7 +121,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -102,20 +131,12 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((&ageconsent=ok)?&warning=\d+)'",data)
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -123,10 +144,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 # link and reload data.
                 addurl = m.group(1)
                 # correct stupid & error in url.
-                # explicitly put ageconsent because google appengine regexp doesn't include it for some reason.
-                addurl = addurl.replace("&","&")+'&ageconsent=ok'
+                addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -141,17 +161,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
-        # because for some reason, this works while simple 'print data' errors on ascii conversion.
-        # loopdata = data
-        # chklen=5000
-        # while len(loopdata) > 0:
-        #     if len(loopdata) < 5000:
-        #         chklen = len(loopdata)
-        #     logging.info("loopdata: %s" % loopdata[:chklen])
-        #     loopdata = loopdata[chklen:]
-            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
+        # print data
 
         # Now go hunting for all the meta data and the chapter list.
         
@@ -191,11 +203,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while value and not defaultGetattr(value,'class') == 'label':
+                while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                # sometimes poorly formated desc (<p> w/o </p>) leads
-                # to all labels being included.
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -214,19 +224,19 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
                 for char in chars:
                     self.story.addToList('characters',char.string)
+                    
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-                    
-            if 'Tags' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=7'))
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -240,8 +250,6 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
         try:
@@ -260,6 +268,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -270,7 +279,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','mw') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Queer as Folk") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y" # XXX
@@ -100,7 +96,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -133,7 +129,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -264,6 +260,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -274,7 +271,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -59,11 +60,6 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','mgln') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y" # XXX
@@ -101,13 +97,13 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId')
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -130,7 +126,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -169,7 +165,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -310,6 +306,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -320,7 +317,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ntlb')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","West Wing")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m-%d-%y"
@@ -85,7 +81,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -154,6 +150,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
                 for char in value.string.split(', '):
                     self.story.addToList('characters',char)
                     
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
             if 'Warnings' in label:
                 for warning in value.string.split(', '):
                     self.story.addToList('warnings',warning)
@@ -163,6 +163,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
             
             if 'Series' in label:
                 self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
@@ -178,7 +179,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class NCISFicComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ncisf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m-%d-%y"
@@ -85,7 +81,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -154,6 +150,10 @@ class NCISFicComAdapter(BaseSiteAdapter):
                 for char in value.string.split(', '):
                     self.story.addToList('characters',char)
                     
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
             if 'Warnings' in label:
                 for warning in value.string.split(', '):
                     self.story.addToList('warnings',warning)
@@ -164,6 +164,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
             if 'Series' in label:
                 if "No Series" not in value.nextSibling.string:
                     self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                    self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
@@ -185,7 +186,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -27,15 +28,20 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return NCISFictionComAdapter
+    return NCISFictionNetAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class NCISFictionComAdapter(BaseSiteAdapter):
+class NCISFictionNetAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
+        # ncisfiction.net blocks IPs the default user-agent.  However,
+        # when asked, they said it was just general anti-spam, not
+        # targeted at us.  That lets me do this in good conscience:
+        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
+        
         self.decode = ["iso-8859-1",
                        "Windows-1252"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -47,7 +53,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL("http://"+self.getSiteDomain()\
@@ -56,11 +62,6 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ncisfn')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%Y"
@@ -68,13 +69,21 @@ class NCISFictionComAdapter(BaseSiteAdapter):
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.ncisfiction.com'
+        return 'www.ncisfiction.net'
+
+    ## Changed from www.ncisfiction.com to www.ncisfiction.net Oct
+    ## 2012 due to the ncisfiction.com domain expiring.  Still accept
+    ## .com domains for existing updates, etc.
+    
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ncisfiction.net','www.ncisfiction.com']
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
 
     def getSiteURLPattern(self):
-        return "http://"+self.getSiteDomain()+r'/(chapters|story)?.php\?stid=\d+'
+        return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+'
        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
@@ -83,7 +92,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -157,6 +166,11 @@ class NCISFictionComAdapter(BaseSiteAdapter):
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
+            if 'Pairing' in label:
+                ships = value.findAll('a')
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
             if 'Genre' in label:
                 genres = value.findAll('a')
                 for genre in genres:
@@ -179,7 +193,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
             series_name = a.find('a').string
             i = a.text.split('#')[1]
             self.setSeries(series_name, i)
-            
+            self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href'])
         except:
             # I find it hard to care if the series parsing fails
             pass
@@ -187,7 +201,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','nfa') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y" # XXX
@@ -104,7 +100,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -137,7 +133,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -268,6 +264,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -278,7 +275,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','nha')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Buffy the Vampire Slayer")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = " %m/%d/%y"
@@ -79,7 +75,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -202,7 +198,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
 

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','osph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -99,13 +95,13 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Logout" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -117,7 +113,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -250,7 +246,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
         data = data.replace('<div align="left"', '<div align="left">')

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','odf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","One Direction")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y"
@@ -102,13 +98,13 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -130,7 +126,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -248,6 +244,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -258,7 +255,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2, urllib, cookielib
 
@@ -47,7 +48,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
@@ -55,11 +56,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','phs')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d %Y"
@@ -95,12 +91,12 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         params['login'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/users/processlogin.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['txtusername']))
         d = self._fetchUrl(loginUrl, params)
     
         if 'Please login to continue.' in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['txtusername']))
             raise exceptions.FailedToLogin(url,params['txtusername'])
             return False
@@ -113,7 +109,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -213,7 +209,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
 	# grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PommeDeSangComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PommeDeSangComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # pommedesang.com has two 'sections', shown in URL as
+        # 'efiction' and 'sds' that change how things should be
+        # handled.
+        # http://pommedesang.com/efiction/viewstory.php?sid=1234
+        # http://pommedesang.com/sds/viewstory.php?sid=1234
+        self.section=self.parsedUrl.path.split('/',)[1]
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pmds')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'efiction' in self.section:
+            self.dateformat = "%b %d, %Y"
+        else:
+            self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'pommedesang.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/sds/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(efiction|sds)?/viewstory.php\?sid=\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/'+self.section+'/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile('viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        logging.info(self.parsedUrl.netloc)
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.info(self.parsedUrl.netloc)
         # normalized story URL.
         if "explicit" in self.parsedUrl.netloc:
             self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -60,11 +61,6 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','pffa')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","My Little Pony")
-            
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
@@ -96,7 +92,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -117,7 +113,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -229,6 +225,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -239,10 +236,9 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
@@ -54,7 +55,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
@@ -62,11 +63,6 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','prtky') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y" # XXX
@@ -86,7 +82,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # portkey screws around with using a different URL to set the
         # cookie and it's a pain.  So... cheat!
@@ -252,7 +248,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,11 +38,10 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -64,7 +64,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -142,7 +142,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
                 charstext = [char.string for char in chars]
                 for char in charstext:
-                    if char == "!Snape and Harry (required)":
+                    if "Snape and Harry (required)" in char:
                         self.story.addToList('characters',"Snape")
                         self.story.addToList('characters',"Harry")
                     else:
@@ -162,12 +162,12 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%b %d %Y"))
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
             
             if 'Updated' in label:
                 # there's a stray [ at the end.
                 #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.
@@ -182,6 +182,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -192,7 +193,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_potterficscom.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return PotterFicsComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterFicsComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL. gets rid of chapter if there, left with chapter index URL
+            nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','potficscom')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.potterfics.com'
+
+    def getSiteExampleURLs(self):
+        return \
+                "http://www.potterfics.com/historias/127583 "\
+                "http://www.potterfics.com/historias/127583/capitulo-1 "\
+                "http://www.potterfics.com/historias/127583/capitulo-4 "\
+                "http://www.potterfics.com/historias/92810 "\
+                "http://www.potterfics.com/historias/111194"
+
+    def getSiteURLPattern(self):
+        #http://www.potterfics.com/historias/127583
+        #http://www.potterfics.com/historias/127583/capitulo-1
+        #http://www.potterfics.com/historias/127583/capitulo-4
+        #http://www.potterfics.com/historias/92810              -> Complete story
+        #http://www.potterfics.com/historias/111194             -> Complete, single chap
+        p = re.escape("http://"+self.getSiteDomain()+"/historias/")+\
+            r"(?P<id>\d+)(/capitulo-(?P<ch>\d+))?/?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        #this converts '/historias/12345' to 'http://www.potterfics.com/historias/12345'
+        def makeAbsoluteURL(url):
+            if url[0] == '/':
+                url = 'http://'+self.getSiteDomain()+url
+            return url
+
+        #use this to get month numbers from Spanish months
+        SpanishMonths = {
+            'enero'      : '01',
+            'febrero'    : '02',
+            'marzo'      : '03',
+            'abril'      : '04',
+            'mayo'       : '05',
+            'junio'      : '06',
+            'julio'      : '07',
+            'agosto'     : '08',
+            'septiembre' : '09',
+            'octubre'    : '10',
+            'noviembre'  : '11',
+            'diciembre'  : '12'
+            }
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        #deal with adult content warnings - doesn't seem to apply to this site
+
+        #set constant meta for this site:
+        #Set Language = Spanish
+        self.story.setMetadata('language', 'Spanish')
+        #Set Category = Harry Potter
+        #  This is better done in plugin-defaults.ini and defaults.ini
+        #  by adding a section for this site with the line:
+        #  extracategories:Harry Potter
+        #self.story.addToList('category','Harry Potter')
+        
+        #get the rest of the meta
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        #self closing br and img present!
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','img'))
+    
+        #we want the second table directly under the body, contains all the metadata
+        table = soup.html.body.findAll('table', recursive=False)[1]
+        #within that, we want the second row, first cell
+        cell = table.tr.findNextSibling('tr').td
+
+        #find first metadata block
+        mb = cell.div.findNextSibling('div')
+        #Get meta...
+        self.story.setMetadata('title', mb.b.string)
+        #strip out brackets on rating
+        self.story.setMetadata('rating', mb.span.string[1:-1])
+        #Completion status is denoted by the presence of this image:
+        if mb.find('img',title="Historia terminada"):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        #find next metadata block
+        #author details
+        mb = mb.findNextSibling('div')
+        self.story.setMetadata('author', mb.b.a.string.strip())
+        self.story.setMetadata('authorUrl', makeAbsoluteURL(mb.b.a['href']))
+        self.story.setMetadata('authorId', self.story.getMetadata('authorUrl').split('/')[4])
+        #dates and times
+        mb = mb.find('span')
+        #posted/published = Escrita
+        date = mb.find(text=re.compile('Escrita el ')).strip().split()
+        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
+        month = int(SpanishMonths[date[5].lower()])
+        day = int(date[3])
+        time = date[8].split(':')
+        hour = int(time[0])
+        minute = int(time[1])
+        self.story.setMetadata('datePublished', datetime.datetime(year, month, day, hour, minute))
+        #updated = Actualizada
+        date = mb.find(text=re.compile('Actualizada el ')).strip().split()
+        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
+        month = int(SpanishMonths[date[5].lower()])
+        day = int(date[3])
+        time = date[8].split(':')
+        hour = int(time[0])
+        minute = int(time[1])
+        self.story.setMetadata('dateUpdated', datetime.datetime(year, month, day, hour, minute))
+        
+        mb = mb.span.findNextSibling('span').findNextSibling('span')
+        wc = mb.find(text=re.compile(' palabras en total')).strip()
+        self.story.setMetadata('numWords', wc.split()[0])
+        
+        #then we come to categories and genres. Oh dear. On this site, categories hold everything from genre, to ships, to crossovers.
+        #To make things worse, there is also another genre field, which often holds similar/duplicate info. Links to genre pages do not work
+        #though, so perhaps those will be phased out?
+        #for now, put them all into the genre list
+        links = mb.findAll('a',href=re.compile('/(categorias|generos)/\d+'))
+        genlist = [i.string.strip() for i in links]
+        self.story.extendList('genre',genlist)
+        
+        #get the chapter urls
+        #we can go back to the table cell we found before
+        #get its last element and work backwards to find the last ordered list on the page
+        list = cell.contents[len(cell)-1].findPrevious('ol')
+        chapters = []
+        revs = 0
+        chnum = 0
+        for li in list:
+            chnum += 1
+            chTitle = str(chnum) + '. ' + li.a.b.string.strip()
+            chURL = makeAbsoluteURL(li.a['href'])
+            chapters.append((chTitle,chURL))
+            #Get reviews, add to total
+            revs += int(li.div.a.string.split()[0])
+        
+        self.chapterUrls.extend(chapters)
+        self.story.setMetadata('numChapters', len(chapters))
+        self.story.setMetadata('reviews', revs)
+        
+        #Now for the description... this may be tricky...
+        #if it is there (doesn't have to be), it will be before the chapter list,
+        #separated by a horizontal rule, and after the google ad bar
+        
+        #get list's parent div
+        mb = list.parent
+        #get the div before that, will either be the description, or the google ad bar
+        mb = mb.findPreviousSibling('div')
+        if 'google_ad_client' in str(mb):
+            #couldn't find description, leaving it blank
+            pass
+        else:
+            self.setDescription(url,mb)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr','img'))
+        
+        div = soup.find('div', {'id' : 'cuerpoHistoria'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ptdc')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","The Pretender")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y"
@@ -94,7 +90,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -114,7 +110,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -232,6 +228,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -242,7 +239,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_prisonbreakficnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','pbf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Prison Break")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"
@@ -81,7 +77,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -157,6 +153,11 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
+                for ship in ships:
+                    self.story.addToList('ships',ship.string.replace(" and ","/"))
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
                 for genre in genres:
@@ -192,6 +193,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -202,7 +204,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_psychficcom.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PsychFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PsychFicComAdapter(BaseSiteAdapter): 
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','psyf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.psychfic.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_qafficcom.py

@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return QafFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class QafFicComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','atp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.qaf-fic.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=")+r"\d+$"
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=NC-17"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\s+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/atp/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. 
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification' and value != None:
+                    if "Featured Stories" not in value:
+                        svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            if list.find('a', href=re.compile(r"series.php")) != None:
+                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
+                    # Find Series name from series URL.
+                    series_url = 'http://'+self.host+'/atp/'+series['href']
+                    # use BeautifulSoup HTML parser to make everything easier to find.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+                    i=1
+                    for a in storyas:
+                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                            name=seriessoup.find('div', {'id' : 'pagetitle'})
+                            name.find('a').extract()
+                            self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
+                            i=0
+                            break
+                        i+=1
+                    if i == 0:
+                        break
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+from datetime import datetime
+import json
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return RestrictedSectionOrgSiteAdapter
+
+class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
+    
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+            
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        
+        # normalized story URL.
+        # get story/file and storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL('http://' + self.getSiteDomain() + '/' + m.group('filestory') + '.php?' + m.group('filestory') + '=' + self.story.getMetadata('storyId'))
+            logger.debug("storyUrl: (%s)"%self.story.getMetadata('storyUrl'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        self.story.setMetadata('siteabbrev','ressec')
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y" # 20 Nov 2005
+        
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'www.restrictedsection.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?story=1234 http://"+self.getSiteDomain()+"/file.php?file=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/(?P<filestory>file|story).php\?(file|story)=(?P<id>\d+)$"
+        
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        # one-shot stories use file url instead of story.  'Luckily',
+        # we don't have to worry about one-shots becoming
+        # multi-chapter because ressec is frozen.  Still need 'story'
+        # url for metadata, however.
+        try:
+            if 'file' in url:
+                data = self._postUrlUP(url)
+                soup = bs.BeautifulSoup(data)
+                storya = soup.find('a',href=re.compile(r"^story.php\?story=\d+"))
+                url = 'http://'+self.host+'/'+storya['href'].split('&')[0]  # strip rs_session
+
+                fileas = soup.find('a',href=re.compile(r"^file.php\?file=\d+"))
+                if fileas:
+                    for filea in fileas:
+                        if 'Previous Chapter' in filea.string or 'Next Chapter' in filea.string:
+                            raise exceptions.FailedToDownload(self.getSiteDomain() +" Cannot use chapter url with multi-chapter stories on this site.")
+                    
+            logger.debug("metadata URL: "+url)
+            data = self._fetchUrl(url)
+            # print data
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+        if "Story not found" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Story not found.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # check user/pass on a chapter for multi-chapter
+        if 'file' not in self.url:
+            self._postUrlUP('http://'+self.host+'/'+soup.find('a', href=re.compile(r"^file.php\?file=\d+"))['href'])
+
+        ## Title
+        h2 = soup.find('h2')
+
+        # Find authorid and URL from... author url.
+        a = h2.find('a')
+        ahref = a['href'].split('&')[0]  # strip rs_session
+        
+        self.story.setMetadata('authorId',ahref.split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref)
+        self.story.setMetadata('author',a.string)
+
+        # title, remove byauthorname.
+        self.story.setMetadata('title',h2.text[:h2.text.index("by"+a.string)])
+
+        dates = soup.findAll('span', {'class':'date'})
+        if dates: # only for multi-chapter
+            self.story.setMetadata('datePublished', makeDate(stripHTML(dates[0]), self.dateformat))
+            self.story.setMetadata('dateUpdated', makeDate(stripHTML(dates[-1]), self.dateformat))
+
+        words = soup.findAll('span', {'class':'size'})
+        wordcount=0
+        for w in words:
+            wordcount = wordcount + int(w.string[:-6].replace(',',''))
+
+        self.story.setMetadata('numWords',"%s"%wordcount)
+
+        self.story.setMetadata('rating', soup.find('a',href=re.compile(r"^rating.php\?rating=\d+")).string)
+        
+        # other tags
+
+        labels = soup.find('table', {'class':'info'}).findAll('th')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if label != None:
+                
+                if 'Categories' in label:
+                    for g in stripHTML(value).split('\n'):
+                        self.story.addToList('genre',g)
+                
+                if 'Pairings' in label:
+                    for g in stripHTML(value).split('\n'):
+                        self.story.addToList('ships',g)
+                
+                if 'Summary' in label:
+                    self.setDescription(url,stripHTML(value).replace("\n"," ").replace("\r",""))
+                    value.extract() # remove summary incase it contains file URLs.
+
+                if 'Updated' in label: # one-shots only.
+                    print "value:%s"%value
+                    value.find('sup').extract() # remove 'st', 'nd', 'th' ordinals
+                    print "value:%s"%value
+                    date = makeDate(stripHTML(value), '%d %B %Y') # full month name
+                    self.story.setMetadata('datePublished', date)
+
+                if 'Length' in label: # one-shots only.
+                    self.story.setMetadata('numWords',value.string[:-6])
+                    
+        # one-shot.
+        if 'file' in self.url:
+            self.chapterUrls.append((self.story.getMetadata('title'),self.url))
+        else: # multi-chapter
+            # Find the chapters: 'library_storyview.php?chapterid=3
+            chapters=soup.findAll('a', href=re.compile(r"^file.php\?file=\d+"))
+            if len(chapters)==0:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No chapters found.")
+            else:
+                for chapter in chapters:
+                    chhref = chapter['href'].split('&')[0] # strip rs_session
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chhref))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+
+    def _postUrlUP(self, url):
+        params = {}
+        if self.password:
+            params['username'] = self.username
+            params['password'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['accept.x'] = 1
+        params['accept.y'] = 1
+
+        excpt=None
+        for sleeptime in [0.5, 1.5, 4, 9]:
+            time.sleep(sleeptime)	
+            try:
+                data = self._postUrl(url, params)
+                if data == "Unable to connect to the database":
+                    raise exceptions.FailedToDownload("Site reported 'Unable to connect to the database'")
+                if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data:
+                    raise exceptions.FailedToLogin(url,params['username'])
+                return data
+            except exceptions.FailedToLogin, ftl:
+                # no need to retry these.
+                raise(ftl)
+            except Exception, e:
+                excpt=e
+                logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
+
+        logger.error("Giving up on %s" %url)
+        logger.exception(excpt)
+        raise(excpt)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        data = self._postUrlUP(url)
+        #print("data:%s"%data)
+        
+        # some stories have html that confuses the parser.  For story
+        # text we don't care about anything before '<table id="page"'
+        # and seems to clear the issue.
+        data = data[data.index('<table id="page"'):]
+        
+        soup = bs.BeautifulSoup(data)
+        
+        div = soup.find('td',{'id':'page_content'})
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        ## Remove stuff from page_content
+
+        # Remove all tags before the first <hr> after class=info table (including hr)
+        hr = div.find('table',{'class':'info'}).findNext('hr')
+        for tag in hr.findAllPrevious():
+            tag.extract()
+        hr.extract()
+        
+        # Remove all tags after the last <hr> (including hr)
+        hr = div.findAll('hr')[-1]
+        for tag in hr.findAllNext():
+            tag.extract()
+        hr.extract()
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','sda')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Supernatural")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y"
@@ -85,7 +81,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -211,6 +207,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -221,7 +218,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,13 +56,6 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','scacf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Glee")
-        self.story.addToList("characters","Kurt Hummel")
-        self.story.addToList("characters","Blaine Anderson")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -101,7 +95,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -121,7 +115,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -228,6 +222,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -238,7 +233,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,19 +48,19 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.section=self.parsedUrl.path.split('/',)[1]
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/'+self.story.getMetadata('section')+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','sghp')
 
         # If all stories from the site fall into the same category,
         # the site itself isn't likely to label them as such, so we
-        # do.
-        if 'atlantis' in self.story.getMetadata('section'):
+        # do.  Can't use extracategories, could be Atlantis or SG-1
+        if 'atlantis' in self.section:
             self.story.addToList("category","Stargate: Atlantis")
         else:
             self.story.addToList("category","Stargate: SG-1")
@@ -95,7 +96,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -115,7 +116,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -149,7 +150,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -220,7 +221,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -232,6 +233,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -242,7 +244,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','snds')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Supernatural")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -90,7 +86,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -110,7 +106,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -230,6 +226,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -240,7 +237,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_siyecouk.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -56,11 +57,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','siye') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d" # XXX
@@ -87,7 +83,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         # sites skip that for one-chapter stories.
         # Except it doesn't this time. :-/
         url = self.url #+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -219,6 +215,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -229,7 +226,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         # soup = bs.BeautifulSoup(self._fetchUrl(url))
         # BeautifulSoup objects to <p> inside <span>, which

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -58,7 +59,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -75,7 +76,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.squidge.org'
     
-    @classmethod # must be @staticmethod, don't remove it.
+    @classmethod # must be @classmethod, don't remove it.
     def getConfigSection(cls):
         # The config section name.  Only override if != site domain.
         return cls.getSiteDomain()+'/peja'
@@ -90,7 +91,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -156,6 +157,8 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
                 #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
+                if value.endswith("["):
+                    value = value[:-1]
                 self.story.setMetadata('rating', value)
 
             if 'Characters' in label:
@@ -209,6 +212,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
                 
@@ -221,7 +225,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','stat')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Stargate: Atlantis")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d %Y"
@@ -81,7 +77,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -208,6 +204,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -218,7 +215,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','soa')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Lord of the Rings")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -81,7 +77,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -138,7 +134,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_svufictioncom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -92,13 +93,13 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -120,7 +121,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -145,7 +146,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -248,6 +249,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -258,7 +260,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -43,7 +44,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -81,13 +82,13 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -102,7 +103,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -116,7 +117,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
             # need to log in for this one.
             addurl = "&ageconsent=ok&warning=4"
             url = self.url+'&index=1'+addurl
-            logging.debug("Changing URL: "+url)
+            logger.debug("Changing URL: "+url)
             self.performLogin(url)
             data = self._fetchUrl(url)
 
@@ -219,6 +220,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -229,7 +231,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_test1.py

@@ -18,6 +18,7 @@
 import datetime
 import time
 import logging
+logger = logging.getLogger(__name__)
 
 from .. import BeautifulSoup as bs
 from .. import exceptions
@@ -48,7 +49,7 @@ class TestSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         if self.story.getMetadata('storyId') == '665' and not (self.is_adult or self.getConfig("is_adult")):
-            logging.warn("self.is_adult:%s"%self.is_adult)
+            logger.warn("self.is_adult:%s"%self.is_adult)
             raise exceptions.AdultCheckRequired(self.url)
 
         if self.story.getMetadata('storyId') == '666':
@@ -73,8 +74,8 @@ class TestSiteAdapter(BaseSiteAdapter):
             self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
             self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
-        self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done
-
+        self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
+<p>
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
         self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
@@ -100,9 +101,12 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                 }
             self.story.setMetadata('language',langs[idnum%len(langs)])
             self.setSeries('The Great Test',idnum)
-        
+            self.story.setMetadata('seriesUrl','http://test1.com?seriesid=1')
+        if idnum == 0:
+            self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum)
+            self.story.setMetadata('seriesUrl','http://test1.com?seriesid=0')
+            
         self.story.setMetadata('rating','Tweenie')
-        
 
         if self.story.getMetadata('storyId') == '673':
             self.story.addToList('author','Author From List')
@@ -113,6 +117,10 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         
             self.story.addToList('authorUrl','http://author/url')
             self.story.addToList('authorUrl','http://author/url-2')
+            self.story.addToList('category','Power Rangers')
+            self.story.addToList('category','SG-1')
+            self.story.addToList('genre','Porn')
+            self.story.addToList('genre','Drama')
         else:
             self.story.setMetadata('authorId','98765')
             self.story.setMetadata('authorUrl','http://author/url')
@@ -120,29 +128,64 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('warnings','Swearing')
         self.story.addToList('warnings','Violence')
 
-        self.story.addToList('category','Harry Potter')
-        self.story.addToList('category','Furbie')
-        self.story.addToList('category','Crossover')
-        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
-        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
-
+        if self.story.getMetadata('storyId') == '80':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+        elif self.story.getMetadata('storyId') == '81':
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+        elif self.story.getMetadata('storyId') == '83':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+        elif self.story.getMetadata('storyId') == '82':
+            self.story.addToList('characters','Henry (Once Upon a Time)')        
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+        else:
+            self.story.addToList('category','Harry Potter')
+            self.story.addToList('category','Furbie')
+            self.story.addToList('category','Crossover')
+            self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+            self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+            self.story.addToList('characters','Bob Smith')
+            self.story.addToList('characters','George Johnson')
+            self.story.addToList('characters','Fred Smythe')
+            
         self.story.addToList('genre','Fantasy')
-        self.story.addToList('genre','SF')
+        self.story.addToList('genre','Comedy')
+        self.story.addToList('genre','Sci-Fi')
         self.story.addToList('genre','Noir')
+                
+        self.story.addToList('listX','xVal1')
+        self.story.addToList('listX','xVal2')
+        self.story.addToList('listX','xVal3')
+        self.story.addToList('listX','xVal4')
         
-        self.story.addToList('characters','Bob Smith')
-        self.story.addToList('characters','George Johnson')
-        self.story.addToList('characters','Fred Smythe')
+        self.story.addToList('listY','yVal1')
+        self.story.addToList('listY','yVal2')
+        self.story.addToList('listY','yVal3')
+        self.story.addToList('listY','yVal4')
         
+        self.story.addToList('listZ','zVal1')
+        self.story.addToList('listZ','zVal2')
+        self.story.addToList('listZ','zVal3')
+        self.story.addToList('listZ','zVal4')
+        
+        self.story.setMetadata('metaA','98765')
+        self.story.setMetadata('metaB','01245')
+        self.story.setMetadata('metaC','The mighty metaC!')
+
         self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
                             ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
                             ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"),
                             ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
-                            #('Chapter 6',self.url+"&chapter=7"),
-                            #('Chapter 7',self.url+"&chapter=8"),
-                            #('Chapter 8',self.url+"&chapter=9"),
+                            ('Chapter 6',self.url+"&chapter=7"),
+                            ('Chapter 7',self.url+"&chapter=8"),
+                            ('Chapter 8',self.url+"&chapter=9"),
                             #('Chapter 9',self.url+"&chapter=0"),
                             #('Chapter 0',self.url+"&chapter=a"),
                             #('Chapter a',self.url+"&chapter=b"),
@@ -164,10 +207,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
-        if self.story.getMetadata('storyId') == '667':
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
-
+        logger.debug('Getting chapter text from: %s' % url)
         if self.story.getMetadata('storyId').startswith('670') or \
                 self.story.getMetadata('storyId').startswith('672'):
             time.sleep(1.0)
@@ -180,20 +220,40 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
-<p>http://test1.com?sid=667 - raises FailedToDownload on chapter 1</p>
+<p>http://test1.com?sid=667 - raises FailedToDownload on chapters 2+</p>
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
+
+
+
+
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
-<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
+<p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p>
+<p>http://test1.com?sid=0 - Succeeds, generates some text specifically for testing hyphenation problems with Nook STR/STRwG</p>
 <p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
+'''
+        elif self.story.getMetadata('storyId') == '0':
+            text=u'''
+<h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
+<br />
+    eyes… but I’m not convinced we should automatically<br />
+<br /><br />
+<b>Thanks to the latest to recommend me: Alastor</b><br />
+<br /><br />
+    “Sure, invite her along. Does she have children?”<br />
+<br />
 '''
         else:
+            if self.story.getMetadata('storyId') == '667':
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
+
             text=u'''
 <div>
 <h3>Chapter title from site</h3>
+<p>Timestamp:'''+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+'''</p>
 <p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 br breaks<br><br>
 Puella Magi Madoka Magica/魔法少女まどか★マギカ

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','tag')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Stargate: SG-1")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d %b %Y"
@@ -81,7 +77,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -193,6 +189,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -203,7 +200,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','thf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d"
@@ -85,7 +81,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -151,6 +147,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
             self.story.setMetadata('status', 'In-Progress')
 						
         summary = info.find('td', {'class' : 'summary'})
+        summary.name='div' # change td to div so it doesn't mess up the display when using table titlepage.
         self.setDescription(url,summary)
 		
         rating=stripHTML(info.find('td', {'align' : 'left'})).split('(')[1].split(')')[0]
@@ -187,7 +184,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr','img')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_thehookupzonenet.py

@@ -0,0 +1,307 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return TheHookupZoneNetAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/CriminalMinds/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','thupz') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thehookupzone.net' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/CriminalMinds/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/CriminalMinds/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/CriminalMinds/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_themasquenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,20 +48,20 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         if self.parsedUrl.path.split('/',)[1] == 'wiktt':
             self.story.addToList("category","Harry Potter")
-            self.story.setMetadata('section','/wiktt/efiction/')
+            self.section='/wiktt/efiction/'
             self.dateformat = "%m/%d/%Y"
         else:
             self.story.addToList("category","Originals")
-            self.story.setMetadata('section','/efiction/')
+            self.section='/efiction/'
             self.dateformat = "%b %d, %Y"
             
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + self.story.getMetadata('section') + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + self.section + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','msq')
@@ -98,14 +99,14 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         params['cookiecheck'] = '1'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain()  + self.story.getMetadata('section') + 'user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        loginUrl = 'http://' + self.getSiteDomain()  + self.section + 'user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -127,7 +128,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -152,7 +153,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -186,7 +187,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.story.getMetadata('section') + chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.section + chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -258,7 +259,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_thepetulantpoetesscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
@@ -91,13 +92,13 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "My Account Page" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -110,7 +111,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -223,7 +224,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the  part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','tqdpch') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y" # XXX
@@ -106,13 +102,13 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -134,7 +130,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -267,6 +263,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -277,7 +274,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -43,11 +44,11 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d %b %Y"
 
             
     @staticmethod
@@ -81,13 +82,13 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/library/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -102,7 +103,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -226,6 +227,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -235,7 +237,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time
@@ -40,7 +41,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL("http://"+self.getSiteDomain()\
                          +"/Story-"+self.story.getMetadata('storyId'))
@@ -81,7 +82,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             return
         
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['urealname']))
 
         ## need to pull empty login page first to get ctkn and
@@ -98,9 +99,9 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         d = self._fetchUrl(loginUrl, params)
     
         if "Stories Published" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                             params['urealname']))
+            raise exceptions.FailedToLogin(self.url,params['urealname'])
             return False
         else:
             return True
@@ -110,7 +111,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         # metadata and chapter list
 
         url=self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # tth won't send you future updates if you aren't 'caught up'
         # on the story.  Login isn't required for F21, but logging in will
@@ -137,7 +138,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 form = soup.find('form', {'id':'sitemaxratingform'})
                 params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
                         'sitemaxrating':'5'}
-                logging.info("Attempting to get rating cookie for %s" % url)
+                logger.info("Attempting to get rating cookie for %s" % url)
                 data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
                 # refetch story page.
                 data = self._fetchUrl(url)
@@ -158,31 +159,58 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             try:
                 # going to pull part of the meta data from author list page.
                 infourl = 'http://'+self.host+ainfo['href']
-                logging.debug("**StoryInfo** URL: "+infourl)
+                logger.debug("**StoryInfo** URL: "+infourl)
                 infodata = self._fetchUrl(infourl)
                 infosoup = bs.BeautifulSoup(infodata)
 
-                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
-                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
-                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
-                    self.story.addToList('author',stripHTML(a))                
+                # for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                #     self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                #     self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                #     self.story.addToList('author',stripHTML(a))
+
+                # second verticaltable is the chapter list.
+                table = infosoup.findAll('table',{'class':'verticaltable'})[1]
+                for a in table.findAll('a',href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))):
+                    autha = a.findNext('a',href=re.compile(r"^/Author-\d+"))
+                    self.story.addToList('authorId',autha['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+autha['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(autha))
+                    # include leading number to match 1. ... 2. ...
+                    self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1,
+                                                             stripHTML(a),
+                                                             stripHTML(autha)),'http://'+self.host+a['href']))
                                             
             except urllib2.HTTPError, e:
                 if e.code == 404:
                     raise exceptions.StoryDoesNotExist(url)
                 else:
                     raise e
+        else: # single author:
+            # Find the chapter selector 
+            select = soup.find('select', { 'name' : 'chapnav' } )
+        	 
+            if select is None:
+        	   # no selector found, so it's a one-chapter story.
+        	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+            else:
+                allOptions = select.findAll('option')
+                for o in allOptions:
+                    url = "http://"+self.host+o['value']
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(o),url))
     
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
         try:
             # going to pull part of the meta data from *primary* author list page.
-            logging.debug("**AUTHOR** URL: "+authorurl)
+            logger.debug("**AUTHOR** URL: "+authorurl)
             authordata = self._fetchUrl(authorurl)
             descurl=authorurl
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
                 nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
-                logging.debug("**AUTHOR** nextpage URL: "+nextpage)
+                logger.debug("**AUTHOR** nextpage URL: "+nextpage)
                 authordata = self._fetchUrl(nextpage)
                 descurl=nextpage
                 authorsoup = bs.BeautifulSoup(authordata)
@@ -234,32 +262,15 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         if BtVS:
             self.story.addToList('category','Buffy: The Vampire Slayer')
             
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapnav' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = "http://"+self.host+o['value']
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(o),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
         pseries = soup.find('p', {'style':'margin-top:0px'})
         m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
                      pseries.text)
         if m:
             self.setSeries(m.group('series'),m.group('num'))
-        
-        return
-
+            self.story.setMetadata('seriesUrl',"http://"+self.host+pseries.find('a')['href'])
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulSoup(self._fetchUrl(url))
 
         div = soup.find('div', {'id' : 'storyinnerbody'})

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL. http://www.twilightarchives.com/read/9353
         self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','twa')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Twilight")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d %b %y"
@@ -82,7 +78,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -167,6 +163,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
                 for a in storyas:
                     if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
                     
@@ -177,7 +174,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_twilightednet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,13 +38,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Twilight")
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -84,13 +84,13 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -100,7 +100,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -217,6 +217,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
         
@@ -226,7 +227,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,14 +38,13 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Twilight")
         self.is_adult = False
         self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
         self.password = ""
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -85,13 +85,13 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -110,7 +110,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -139,12 +139,14 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
+        pagetitlediv = soup.find('div',id='pagetitle')
+        
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        a = pagetitlediv.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitlediv.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
@@ -244,6 +246,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -253,7 +256,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_walkingtheplankorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -45,7 +46,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -53,11 +54,6 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','wtp')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y"
@@ -89,7 +85,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -210,6 +206,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -220,7 +217,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_whoficcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -56,7 +57,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         # - get chapter list, if not one-shot.
 
         url = self.url+'&chapter=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
         
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
@@ -69,7 +70,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
 
         # pull title(title) and author from the HTML title.
         title = soup.find('title').string
-        logging.debug('Title: %s' % title)
+        logger.debug('Title: %s' % title)
         title = title.split('::')[1].strip()
         self.story.setMetadata('title',title.split(' by ')[0].strip())
         self.story.setMetadata('author',title.split(' by ')[1].strip())
@@ -109,7 +110,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         # <i>Published:</i> 2010.08.15 - <i>Updated:</i> 2010.08.16 - <i>Chapters:</i> 4 - <i>Completed:</i> Yes - <i>Word Count:</i> 4890 </font>
         # </td></tr></table>
              
-        logging.debug("Author URL: "+self.story.getMetadata('authorUrl'))
+        logger.debug("Author URL: "+self.story.getMetadata('authorUrl'))
         soup = bs.BeautifulStoneSoup(self._fetchUrl(self.story.getMetadata('authorUrl')),
                                      selfClosingTags=('br')) # normalize <br> tags to <br />
         # find this story in the list, parse it's metadata based on
@@ -203,6 +204,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -212,7 +214,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -92,13 +93,13 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -120,7 +121,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -145,7 +146,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -276,6 +277,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -286,7 +288,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.