Bump versions, etc.

ribbon cover exclusion for dramione.org

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-27
+version: 4-4-36
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 6, 11)
+    version             = (1, 7, 2)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/common_utils.py

@@ -196,7 +196,7 @@ class ImageTitleLayout(QHBoxLayout):
     '''
     A reusable layout widget displaying an image followed by a title
     '''
-    def __init__(self, parent, icon_name, title):
+    def __init__(self, parent, icon_name, title, tooltip=None):
         QHBoxLayout.__init__(self)
         title_image_label = QLabel(parent)
         pixmap = get_pixmap(icon_name)
@@ -217,6 +217,9 @@ class ImageTitleLayout(QHBoxLayout):
         self.addWidget(shelf_label)
         self.insertStretch(-1)
 
+        if tooltip:
+            title_image_label.setToolTip(tooltip)
+            shelf_label.setToolTip(tooltip)
 
 class SizePersistedDialog(QDialog):
     '''

calibre-plugin/config.py

@@ -7,20 +7,24 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback, copy
+import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
+                      QDialogButtonBox )
 
 from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order)
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
+    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
+            EditTextDialog)
+    
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
+    import (getConfigSections, getNormalStoryURL)
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
@@ -34,6 +38,10 @@ PREFS_KEY_SETTINGS = 'settings'
 # take from here.
 default_prefs = {}
 default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
 
 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
@@ -137,8 +145,86 @@ class PrefsFacade():
     def save_to_db(self):
         set_library_config(self._get_prefs())
         
-
 prefs = PrefsFacade(default_prefs)
+
+class RejectURLList:
+    def __init__(self,prefs):
+        self.prefs = prefs
+        self.sync_lock = threading.RLock()
+        self.listcache = None
+
+    def _read_list_from_text(self,text,addreasontext=None):
+        cache = {}
+        for line in text.splitlines():
+            if ',' in line:
+                (rejurl,note) = line.split(',',1)
+            else:
+                (rejurl,note) = (line,'')
+            rejurl = getNormalStoryURL(rejurl)
+            if rejurl:
+                if addreasontext and note:
+                    note = note +" - "+addreasontext
+                elif addreasontext:
+                    note = addreasontext
+                cache[rejurl] = note
+        return cache
+        
+
+    def _get_listcache(self):
+        if self.listcache == None:
+            self.listcache = self._read_list_from_text(prefs['rejecturls'])
+        return self.listcache
+
+    def _save_list(self,listcache):
+        rejectlist = []
+        for url in listcache:
+            rejectlist.append("%s,%s"%(url,listcache[url]))
+            
+        self.prefs['rejecturls'] = '\n'.join(rejectlist)
+        self.prefs.save_to_db()
+        self.listcache = None
+        
+    def clear_cache(self):
+        self.listcache = None
+
+    def check(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                note = listcache[url]
+                return note
+            
+            # not found
+            return None
+
+    def remove(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                del listcache[url]
+                self._save_list(listcache)
+
+    def add_text(self,rejecttext,addreasontext):
+        self.add(self._read_list_from_text(rejecttext,addreasontext).items())
+            
+    def add(self,rejectlist,clear=False):
+        # rejectlist=list of (url,note) tuples.
+        with self.sync_lock:
+            if clear:
+                listcache={}
+            else:
+                listcache = self._get_listcache()
+            for (url,note) in rejectlist:
+                listcache[url]=note
+            self._save_list(listcache)
+
+    def get_list(self):
+        return copy.deepcopy(self._get_listcache())
+            
+    def get_reject_reasons(self):
+        return self.prefs['rejectreasons'].splitlines()
+
+rejecturllist = RejectURLList(prefs)
     
 class ConfigWidget(QWidget):
 
@@ -396,6 +482,27 @@ class BasicTab(QWidget):
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.l.addSpacing(10)        
+
+        horz = QHBoxLayout()
+        
+        self.rejectlist = QPushButton('Edit Reject URL List', self)
+        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
+        self.rejectlist.clicked.connect(self.show_rejectlist)
+        horz.addWidget(self.rejectlist)
+        
+        self.reject_urls = QPushButton('Add Reject URLs', self)
+        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
+        self.reject_urls.clicked.connect(self.add_reject_urls)
+        horz.addWidget(self.reject_urls)
+        
+        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
+        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
+        self.reject_reasons.clicked.connect(self.show_reject_reasons)
+        horz.addWidget(self.reject_reasons)
+        
+        self.l.addLayout(horz)
+        
         self.l.insertStretch(-1)
         
     def set_collisions(self):
@@ -411,7 +518,53 @@ class BasicTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+
+    def show_rejectlist(self):
+        rejectlist = []
+        for (url,note) in rejecturllist.get_list().items():
+            rejectlist.append((None,url,note,note))
+            
+        d = RejectListDialog(self,
+                             rejectlist,
+                             rejectreasons=rejecturllist.get_reject_reasons(),
+                             header="Edit Reject URLs List",
+                             show_delete=False,
+                             show_all_reasons=False)
+        d.exec_()
         
+        if d.result() != d.Accepted:
+            return
+        
+        rejectlist=[]
+        for (bookid,url,note) in d.get_reject_list():
+            rejectlist.append((url,note))
+
+        rejecturllist.add(rejectlist,clear=True)
+        
+    def show_reject_reasons(self):
+        d = EditTextDialog(self,
+                           prefs['rejectreasons'],
+                           icon=self.windowIcon(),
+                           title="Reject Reasons",
+                           label="Customize Reject List Reasons",
+                           tooltip="Customize the Reasons presented when Rejecting URLs")
+        d.exec_()
+        if d.result() == d.Accepted:
+            prefs['rejectreasons'] = d.get_plain_text()
+                
+    def add_reject_urls(self):
+        d = EditTextDialog(self,
+                           "http://example.com?story.php?sid=5,Reason why I rejected it",
+                           icon=self.windowIcon(),
+                           title="Add Reject URLs",
+                           label="Add Reject URLs. Use: <b>http://...,note</b><br>Invalid story URLs will be ignored.",
+                           tooltip="One URL per line, everything after <b>,</b> will be put in the note.",
+                           rejectreasons=rejecturllist.get_reject_reasons(),
+                           reasonslabel='Add this reason to all URLs added:')
+        d.exec_()
+        if d.result() == d.Accepted:
+            rejecturllist.add_text(d.get_plain_text(),d.get_reason_text())
+                
 class PersonalIniTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -451,7 +604,7 @@ class PersonalIniTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-
+        
 class ShowDefaultsIniDialog(QDialog):
 
     def __init__(self, icon, text, parent=None):
@@ -916,3 +1069,4 @@ class StandardColumnsTab(QWidget):
             self.l.addLayout(horz)
         
         self.l.insertStretch(-1)
+

calibre-plugin/dialogs.py

@@ -8,15 +8,19 @@ __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import traceback
+from functools import partial
 
 from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
-                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView )
+from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout,
+                      QGridLayout, QPushButton, QProgressDialog, QString, QLabel,
+                      QCheckBox, QIcon, QTextCursor, QTextEdit, QLineEdit, QInputDialog,
+                      QComboBox, QClipboard, QVariant, QProgressDialog, QTimer,
+                      QDialogButtonBox, QPixmap, Qt, QAbstractItemView, SIGNAL,
+                      QTableWidgetItem )
 
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.gui2.complete2 import EditWithComplete
 
 from calibre import confirm_config_name
 from calibre.gui2 import dynamic
@@ -41,6 +45,19 @@ collision_order=[SKIP,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
         
+# This is a more than slightly kludgey way to get
+# EditWithComplete to *not* alpha-order the reasons, but leave
+# them in the order entered.  If
+# calibre.gui2.complete2.CompleteModel.set_items ever changes,
+# this function will need to also.
+def complete_model_set_items_kludge(self, items):
+    items = [unicode(x.strip()) for x in items]
+    items = [x for x in items if x]
+    items = tuple(items)
+    self.all_items = self.current_items = items
+    self.current_prefix = ''
+    self.reset()
+    
 class NotGoingToDownload(Exception):
     def __init__(self,error,icon='dialog_error.png'):
         self.error=error
@@ -712,3 +729,321 @@ class StoryListTableWidget(QTableWidget):
             self.setItem(dest_row, col, self.takeItem(src_row, col))
         self.removeRow(src_row)
         self.blockSignals(False)
+
+class RejectListTableWidget(QTableWidget):
+
+    def __init__(self, parent,rejectreasons=[]):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+        self.rejectreasons = rejectreasons
+
+    def on_headersection_clicked(self):
+        self.setSortingEnabled(True)
+        
+    def populate_table(self, reject_list):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(reject_list))
+        header_labels = ['URL', 'Note']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        # need sortingEnbled to sort, but off to up & down.
+        self.connect(self.horizontalHeader(),
+                     SIGNAL('sectionClicked(int)'),
+                     self.on_headersection_clicked)
+
+        # row is just row number.
+        for row, rejectrow in enumerate(reject_list):
+            self.populate_table_row(row,rejectrow)
+
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(1, 100)
+        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumSize(300, 0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, rejectrow):
+
+        (bookid,url,titleauth,oldrejnote) = rejectrow
+        if oldrejnote:
+            noteprefix = note = oldrejnote
+            # incase the existing note ends with one of the known reasons.
+            for reason in self.rejectreasons:
+                if noteprefix.endswith(' - '+reason):
+                    noteprefix = noteprefix[:-len(' - '+reason)]
+                    break
+        else:
+            noteprefix = note = titleauth
+            
+        if len(noteprefix) > 0:
+            noteprefix = noteprefix+' - '
+            
+        url_cell = ReadOnlyTableWidgetItem(url)
+        url_cell.setData(Qt.UserRole, QVariant(bookid))
+        url_cell.setToolTip('URL to add to the Reject List.')
+        self.setItem(row, 0, url_cell)
+
+        note_cell = EditWithComplete(self)
+
+        note_cell.lineEdit().mcompleter.model().set_items = \
+            partial(complete_model_set_items_kludge,
+                    note_cell.lineEdit().mcompleter.model())
+        
+        items = [note]+[ noteprefix+x for x in self.rejectreasons ]
+        note_cell.update_items_cache(items)
+        note_cell.show_initial_value(note)
+        note_cell.set_separator(None)
+        note_cell.setToolTip('Select or Edit Reject Note.')
+        self.setCellWidget(row, 1, note_cell)
+        
+        # note_cell = QTableWidgetItem(note)
+        # note_cell.setToolTip('Double-click to edit note.')
+        # self.setItem(row, 1, note_cell)
+      
+    def get_reject_list(self):
+        rejectrows = []
+        for row in range(self.rowCount()):
+            bookid = self.item(row, 0).data(Qt.UserRole).toPyObject()
+            url = unicode(self.item(row, 0).text())
+            note = unicode(self.cellWidget(row, 1).currentText()).strip()
+            rejectrows.append((bookid,url,note))
+        return rejectrows
+
+    def remove_selected_rows(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        message = '<p>Are you sure you want to remove this URL from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+        if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
+            return
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
+
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
+
+    def move_rows_up(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        first_sel_row = rows[0].row()
+        if first_sel_row <= 0:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in selrows:
+            self.swap_row_widgets(selrow - 1, selrow + 1)
+        scroll_to_row = first_sel_row - 1
+        if scroll_to_row > 0:
+            scroll_to_row = scroll_to_row - 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def move_rows_down(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        last_sel_row = rows[-1].row()
+        if last_sel_row == self.rowCount() - 1:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in reversed(selrows):
+            self.swap_row_widgets(selrow + 2, selrow)
+        scroll_to_row = last_sel_row + 1
+        if scroll_to_row < self.rowCount() - 1:
+            scroll_to_row = scroll_to_row + 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def swap_row_widgets(self, src_row, dest_row):
+        self.blockSignals(True)
+        self.setSortingEnabled(False)
+        self.insertRow(dest_row)
+        for col in range(0, self.columnCount()):
+            self.setItem(dest_row, col, self.takeItem(src_row, col))
+        self.removeRow(src_row)
+        self.blockSignals(False)
+
+class RejectListDialog(SizePersistedDialog):
+    def __init__(self, gui, reject_list,
+                 rejectreasons=[],
+                 header="List of Books to Reject",
+                 icon='rotate-right.png',
+                 show_delete=True,
+                 show_all_reasons=True,
+                 save_size_name='ffdl:reject list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.gui = gui
+      
+        self.setWindowTitle(header)
+        self.setWindowIcon(get_icon(icon))
+      
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, icon, header,
+                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+        layout.addLayout(title_layout)
+        rejects_layout = QHBoxLayout()
+        layout.addLayout(rejects_layout)
+
+        self.rejects_table = RejectListTableWidget(self,rejectreasons=rejectreasons)
+        rejects_layout.addWidget(self.rejects_table)
+
+        button_layout = QVBoxLayout()
+        rejects_layout.addLayout(button_layout)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        # self.move_up_button = QtGui.QToolButton(self)
+        # self.move_up_button.setToolTip('Move selected books up the list')
+        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
+        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
+        # button_layout.addWidget(self.move_up_button)
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+        # self.move_down_button = QtGui.QToolButton(self)
+        # self.move_down_button.setToolTip('Move selected books down the list')
+        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
+        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
+        # button_layout.addWidget(self.move_down_button)
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+
+        if show_all_reasons:
+            self.reason_edit = EditWithComplete(self)
+            self.reason_edit.lineEdit().mcompleter.model().set_items = \
+                partial(complete_model_set_items_kludge,
+                        self.reason_edit.lineEdit().mcompleter.model())
+            
+            items = ['']+rejectreasons
+            self.reason_edit.update_items_cache(items)
+            self.reason_edit.show_initial_value('')
+            self.reason_edit.set_separator(None)
+            self.reason_edit.setToolTip("This will be added to whatever note you've set for each URL above.")
+            
+            horz = QHBoxLayout()
+            label = QLabel("Add this reason to all URLs added:")
+            label.setToolTip("This will be added to whatever note you've set for each URL above.")
+            horz.addWidget(label)
+            horz.addWidget(self.reason_edit)
+            horz.insertStretch(-1)
+            layout.addLayout(horz)
+                    
+        options_layout = QHBoxLayout()
+
+        if show_delete:
+            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
+            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks.setChecked(True)
+            options_layout.addWidget(self.deletebooks)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+      
+        layout.addLayout(options_layout)
+      
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.rejects_table.populate_table(reject_list)
+
+    def remove_from_list(self):
+        self.rejects_table.remove_selected_rows()
+
+    def get_reject_list(self):
+        return self.rejects_table.get_reject_list()
+
+    def get_reason_text(self):
+        return unicode(self.reason_edit.currentText()).strip()
+    
+    def get_deletebooks(self):
+        return self.deletebooks.isChecked()
+
+class EditTextDialog(QDialog):
+
+    def __init__(self, parent, text,
+                 icon=None, title=None, label=None, tooltip=None,
+                 rejectreasons=[],reasonslabel=None
+                 ):
+        QDialog.__init__(self, parent)
+        self.resize(600, 500)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel(label)
+        if title:
+            self.setWindowTitle(title)
+        if icon:
+            self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.textedit = QTextEdit(self)
+        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setText(text)
+        self.l.addWidget(self.textedit)
+
+        if tooltip:
+            self.label.setToolTip(tooltip)
+            self.textedit.setToolTip(tooltip)
+
+        if rejectreasons or reasonslabel:
+            self.reason_edit = EditWithComplete(self)
+
+            self.reason_edit.lineEdit().mcompleter.model().set_items = \
+                partial(complete_model_set_items_kludge,
+                        self.reason_edit.lineEdit().mcompleter.model())
+            
+            items = ['']+rejectreasons
+            self.reason_edit.update_items_cache(items)
+            self.reason_edit.show_initial_value('')
+            self.reason_edit.set_separator(None)
+            self.reason_edit.setToolTip(reasonslabel)
+            
+            if reasonslabel:
+                horz = QHBoxLayout()
+                label = QLabel(reasonslabel)
+                label.setToolTip(reasonslabel)
+                horz.addWidget(label)
+                horz.addWidget(self.reason_edit)
+                self.l.addLayout(horz)
+            else:
+                self.l.addWidget(self.reason_edit)
+            
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+
+    def get_plain_text(self):
+        return unicode(self.textedit.toPlainText())
+
+    def get_reason_text(self):
+        return unicode(self.reason_edit.currentText()).strip()
+    

calibre-plugin/ffdl_plugin.py

@@ -41,10 +41,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable i
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
 
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
+from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values, rejecturllist)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, 
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
     NotGoingToDownload )
 
@@ -145,6 +145,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def library_changed(self, db):
         # We need to reset our menus after switching libraries
         self.rebuild_menus()
+        rejecturllist.clear_cache()
         
     def rebuild_menus(self):
         with self.menus_lock:
@@ -190,6 +191,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
                                                                 triggered=self.get_urls_from_page)
 
+            self.reject_list_action = self.create_menu_item_ex(self.menu, 'Reject Selected Books', image='rotate-right.png',
+                                                               triggered=self.reject_list_urls)
+
             # print("platform.system():%s"%platform.system())
             # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
             if not self.check_macmenuhack(): # not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
@@ -211,7 +215,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 if menu_id not in self.actions_unique_map:
                     self.gui.keyboard.unregister_shortcut(unique_name)
             self.old_actions_unique_map = self.actions_unique_map
-            self.gui.keyboard.finalize()            
+            self.gui.keyboard.finalize()
 
     def about(self):
         # Get the about text from a file inside the plugin zip file
@@ -238,15 +242,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         #print("create_menu_item_ex after %s"%menu_text)
         return ac
 
+    def is_library_view(self):
+        # 0 = library, 1 = main, 2 = card_a, 3 = card_b
+        return self.gui.stack.currentIndex() == 0
+    
     def plugin_button(self):
-        if len(self.gui.library_view.get_selected_ids()) > 0 and prefs['updatedefault']:
+        if self.is_library_view() and \
+                len(self.gui.library_view.get_selected_ids()) > 0 and \
+                prefs['updatedefault']:
             self.update_existing()
         else:
             self.add_dialog()
 
     def update_lists(self,add=True):
-        if len(self.gui.library_view.get_selected_ids()) > 0 and \
-                (prefs['addtolists'] or prefs['addtoreadlists']) :
+        if prefs['addtolists'] or prefs['addtoreadlists']:
+            if not self.is_library_view():
+                self.gui.status_bar.show_message(_('Cannot Update Reading Lists from Device View'), 3000)
+                return
+        
+            if len(self.gui.library_view.get_selected_ids()) == 0:
+                self.gui.status_bar.show_message(_('No Selected Books to Update Reading Lists'), 3000)
+                return
+
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
 
     def get_urls_from_page(self):
@@ -284,19 +301,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
             
     def get_list_urls(self):
-        if len(self.gui.library_view.get_selected_ids()) > 0:
-            book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
+        if self.gui.current_view().selectionModel().selectedRows() == 0 :
+            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
+                                             3000)
+            return
+            
+        if self.is_library_view():
+            book_list = map( partial(self._convert_id_to_book, good=False),
+                             self.gui.library_view.get_selected_ids() )
 
-            LoopProgressDialog(self.gui,
-                               book_list,
-                               partial(self._get_story_url_for_list, db=self.gui.current_db),
-                               self._finish_get_list_urls,
-                               init_label="Collecting URLs for stories...",
-                               win_title="Get URLs for stories",
-                               status_prefix="URL retrieved")
+        else: # device view, get from epubs on device.
+            view = self.gui.current_view()
+            rows = view.selectionModel().selectedRows()
+            # paths = view.model().paths(rows)
+            book_list = map( partial(self._convert_row_to_book, good=False), rows )
+            
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._get_story_url_for_list, db=self.gui.current_db),
+                           self._finish_get_list_urls,
+                           init_label="Collecting URLs for stories...",
+                           win_title="Get URLs for stories",
+                           status_prefix="URL retrieved")
             
     def _get_story_url_for_list(self,book,db=None):
-        book['url'] = self._get_story_url(db,book['calibre_id'])
+        if book['calibre_id']:
+            book['url'] = self._get_story_url(db,book_id=book['calibre_id'])
+        elif book['path']:
+            book['url'] = self._get_story_url(db,path=book['path'])
+            
         if book['url'] == None:
             book['good']=False
         else:
@@ -314,6 +347,83 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         show=True,
                         show_copy_button=False)
                 
+    def reject_list_urls(self):
+        if self.is_library_view():
+            book_list = map( partial(self._convert_id_to_book, good=False),
+                             self.gui.library_view.get_selected_ids() )
+
+        else: # device view, get from epubs on device.
+            view = self.gui.current_view()
+            rows = view.selectionModel().selectedRows()
+            #paths = view.model().paths(rows)
+            book_list = map( partial(self._convert_row_to_book, good=False), rows )
+            
+        if len(book_list) == 0 :
+            self.gui.status_bar.show_message(_('No Selected Books have URLs to Reject'), 3000)
+            return
+            
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self._reject_story_url_for_list, db=self.gui.current_db),
+                           self._finish_reject_list_urls,
+                           init_label="Collecting URLs for Reject List...",
+                           win_title="Get URLs for Reject List",
+                           status_prefix="URL retrieved")
+            
+    def _reject_story_url_for_list(self,book,db=None): 
+        if book['calibre_id']:
+            # want title/author, too, for rejects.
+            self._populate_book_from_calibre_id(book,db)
+            book['url'] = self._get_story_url(db,book_id=book['calibre_id'])
+        elif book['path']:
+            book['url'] = self._get_story_url(db,path=book['path'])
+                
+        if book['url'] == None:
+            book['good']=False
+        else:
+            book['good']=True
+            # get existing note, if there is one.
+            book['oldrejnote']=rejecturllist.check(book['url'])
+            
+    def _finish_reject_list_urls(self, book_list):
+
+        # construct reject list of tuples:
+        # (calibre_id, url, "title, authors", old reject note).
+        reject_list = [ ( x['calibre_id'],x['url'],
+                          "%s by %s"%(x['title'],
+                                      ', '.join(x['author'])),
+                          x['oldrejnote'])
+                        for x in book_list if x['good'] ]
+        if reject_list:
+            d = RejectListDialog(self.gui,reject_list,
+                                 rejectreasons=rejecturllist.get_reject_reasons())
+            d.exec_()
+            
+            if d.result() != d.Accepted:
+                return
+
+            bookids=[]
+            rejectlist=[]
+            addreasontext=d.get_reason_text()
+            for (bookid,url,note) in d.get_reject_list():
+                bookids.append(bookid)
+                if addreasontext and note:
+                    note = note +" - "+addreasontext
+                elif addreasontext:
+                    note = addreasontext
+                rejectlist.append((url,note))
+                print("Adding (%s) to Reject List: %s"%(url,note))
+                
+            rejecturllist.add(rejectlist)
+
+            if d.get_deletebooks():
+                self.gui.iactions['Remove Books'].delete_books()
+            
+        else:
+            message="<p>Rejecting FFDL URLs: None of the books selected have FanFiction URLs.</p><p>Proceed to Remove?</p>"
+            if confirm(message,'fanfictiondownloader_reject_non_fanfiction', self.gui):
+                self.gui.iactions['Remove Books'].delete_books()
+                
     def add_dialog(self):
 
         #print("add_dialog()")
@@ -346,7 +456,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         self.start_downloads( options, add_books )
         
     def update_existing(self):
+        if not self.is_library_view():
+            self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
+            return
+        
         if len(self.gui.library_view.get_selected_ids()) == 0:
+            self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
             return
         #print("update_existing()")
         
@@ -431,6 +546,29 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         necessary data.  To be called from LoopProgressDialog
         'loop'.  Also pops dialogs for is adult, user/pass.
         '''
+
+        url = book['url']
+        print("url:%s"%url)
+
+        rejnote = rejecturllist.check(url)
+        if rejnote:
+            if question_dialog(self.gui, 'Reject URL?',
+                               '<p>Reject URL?</p>'+
+                               '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
+                               "<p>Click 'No' to download anyway.</p>",
+                               show_copy_button=False):
+                book['comment'] = "Story on Reject URLs list (%s)."%rejnote
+                book['good']=False
+                book['icon']='rotate-right.png'
+                book['status'] = 'Rejected'
+                return
+            else:
+                if question_dialog(self.gui, 'Remove Reject URL?',
+                                   "<p>Remove URL from Reject List?</p>"+
+                                   '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
+                                   "<p>Click 'Yes' to remove it from the list and download,<br /> 'No' to download, but leave it on the Reject list.</p>",
+                                   show_copy_button=False):
+                    rejecturllist.remove(url)
         
         # The current database shown in the GUI
         # db is an instance of the class LibraryDatabase2 from database.py
@@ -447,8 +585,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             # book has already been flagged bad for whatever reason.
             return
         
-        url = book['url']
-        print("url:%s"%url)
         skip_date_update = False
         
         options['personal.ini'] = prefs['personal.ini']
@@ -496,14 +632,12 @@ make_firstimage_cover:true
         book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
         book['tags'] = story.getSubjectTags(removeallentities=True)
-        book['comments'] = sanitize_comments_html(story.getMetadata("description"))        
+        if story.getMetadata("description"):
+            book['comments'] = sanitize_comments_html(story.getMetadata("description"))
+        else:
+            book['comments']=''
         book['series'] = story.getMetadata("series", removeallentities=True)
         
-        # adapter.opener is the element with a threadlock.  But del
-        # adapter.opener doesn't work--subproc fails when it tries
-        # to pull in the adapter object that hasn't been imported yet.
-        # book['adapter'] = adapter
-        
         book['is_adult'] = adapter.is_adult
         book['username'] = adapter.username
         book['password'] = adapter.password
@@ -537,8 +671,8 @@ make_firstimage_cover:true
             # 'new' book from URL.  collision handling applies.
             print("from URL(%s)"%url)
 
-            # try to find by identifier url first.
-            searchstr = 'identifiers:"=url:%s"'%url.replace(":","|")
+            # try to find by identifier url or uri first.
+            searchstr = 'identifiers:"~ur(i|l):=%s"'%url.replace(":","|")
             identicalbooks = db.search_getting_ids(searchstr, None)
             if len(identicalbooks) < 1:
                 # find dups
@@ -634,9 +768,12 @@ make_firstimage_cover:true
             
         if book['good']: # there shouldn't be any !'good' books at this point.
             # if still 'good', make a temp file to write the output to.
-            tmp = PersistentTemporaryFile(prefix='new-%s-'%book['calibre_id'],
-                                               suffix='.'+options['fileform'],
-                                               dir=options['tdir'])
+            # For HTML format users, make the filename inside the zip something reasonable.
+            # For crazy long titles/authors, limit it to 200chars.
+            # For weird/OS-unsafe characters, use file safe only.
+            tmp = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
+                                          suffix='.'+options['fileform'],
+                                          dir=options['tdir'])
             print("title:"+book['title'])
             print("outfile:"+tmp.name)
             book['outfile'] = tmp.name            
@@ -898,13 +1035,13 @@ make_firstimage_cover:true
             # mi.tags needs to be list, but set kills dups.
             mi.tags = list(set(list(old_tags)+mi.tags))
 
-        if 'langcode' in book['all_metadata']:
+        if book['all_metadata']['langcode']:
             mi.languages=[book['all_metadata']['langcode']]
         else:
             # Set language english, but only if not already set.
             if not oldmi.languages:
-                mi.languages=['eng']
-
+                mi.languages=['en']
+                
         if options['fileform'] == 'epub' and prefs['updatecover']:
             existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
             epubmi = get_metadata(existingepub,'EPUB')
@@ -1112,12 +1249,6 @@ make_firstimage_cover:true
                 message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
                 confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
 
-            # Quick demo of how an 'allow send' list might work.
-            # Issues: allow list per send list?  Naming convention?  "send(allow)"
-            # allow_list = rl_plugin.get_book_list("Allow Send to Device")
-            # # intersection of book_ids & allow_list
-            # add_book_ids = list(set(book_ids) & set(allow_list))
-
             for l in lists:
                 if l in rl_plugin.get_list_names():
                     #print("good send l:(%s)"%l)
@@ -1130,17 +1261,6 @@ make_firstimage_cover:true
                         message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
                         confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
 
-    # def _find_existing_book_id(self,db,book,matchurl=True):
-    #     mi = MetaInformation(book["title"],book["author"]) # author is a list.
-    #     identicalbooks = db.find_identical_books(mi)
-    #     if matchurl: # only *really* identical if URL matches, too.
-    #         for ib in identicalbooks:
-    #             if self._get_story_url(db,ib) == book['url']:
-    #                 return ib
-    #     if identicalbooks:
-    #         return identicalbooks.pop()
-    #     return None
-    
     def _make_mi_from_book(self,book):
         mi = MetaInformation(book['title'],book['author']) # author is a list.
         mi.set_identifiers({'url':book['url']})
@@ -1202,7 +1322,25 @@ make_firstimage_cover:true
         book['comment'] = ''
         book['url'] = ''
         book['added'] = False
-        
+
+        return book
+            
+    def _convert_row_to_book(self, row, good=True):
+        book = {}
+        mi = self.gui.current_view().model().get_book_display_info(row.row())
+        book['title'] = mi.title
+        book['author'] = mi.authors
+        book['path'] = mi.path
+        book['author_sort'] = mi.author_sort
+        book['good'] = good
+        book['calibre_id'] = None
+        book['begin'] = None
+        book['end'] = None
+            
+        book['comment'] = ''
+        book['url'] = ''
+        book['added'] = False
+
         return book
             
     def _populate_book_from_calibre_id(self, book, db=None):
@@ -1237,44 +1375,46 @@ make_firstimage_cover:true
                 book['icon']='dialog_error.png'
                 book['status'] = 'Bad URL'
     
-    def _get_story_url(self, db, book_id):
-        identifiers = db.get_identifiers(book_id,index_is_id=True)
+    def _get_story_url(self, db, book_id=None, path=None):
+        if book_id == None:
+            identifiers={}
+        else:
+            identifiers = db.get_identifiers(book_id,index_is_id=True)
         if 'url' in identifiers:
             # identifiers have :->| in url.
-            #print("url from book:"+identifiers['url'].replace('|',':'))
+            # print("url from ident url:%s"%identifiers['url'].replace('|',':'))
             return identifiers['url'].replace('|',':')
+        elif 'uri' in identifiers:
+            # identifiers have :->| in uri.
+            # print("uri from ident uri:%s"%identifiers['uri'].replace('|',':'))
+            return identifiers['uri'].replace('|',':')
         else:
-            ## only epub has URL in it--at least where I can easily find it.
-            if db.has_format(book_id,'EPUB',index_is_id=True):
+            existingepub = None
+            if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                 existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
                 mi = get_metadata(existingepub,'EPUB')
                 identifiers = mi.get_identifiers()
                 if 'url' in identifiers:
-                    #print("url from epub:"+identifiers['url'].replace('|',':'))
+                    # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
                     return identifiers['url'].replace('|',':')
-                # look for dc:source first, then scan HTML if 
+            elif path.lower().endswith('.epub'):
+                existingepub = path
+                
+            ## only epub has URL in it--at least where I can easily find it.
+            if existingepub:                
+                # look for dc:source first, then scan HTML if lookforurlinhtml
                 link = get_dcsource(existingepub)
                 if link:
+                    # print("url from get_dcsource:%s"%link)
                     return link
                 elif prefs['lookforurlinhtml']:
-                    return get_story_url_from_html(existingepub,self._is_good_downloader_url)
+                    link = get_story_url_from_html(existingepub,self._is_good_downloader_url)
+                    # print("url from get_story_url_from_html:%s"%link)
+                    return link
         return None
 
     def _is_good_downloader_url(self,url):
-        # this is the accepted way to 'check for existance of a class variable'?  really?
-        try:
-            self.dummyconfig
-        except AttributeError:
-            self.dummyconfig = Configuration("test1.com","EPUB")
-        # pulling up an adapter is pretty low over-head.  If
-        # it fails, it's a bad url.
-        try:
-            adapter = adapters.getAdapter(self.dummyconfig,url)
-            url = adapter.url
-            del adapter
-            return url
-        except:
-            return None;
+        return adapters.getNormalStoryURL(url)
 
 def get_url_list(urls):
     def f(x):

calibre-plugin/jobs.py

@@ -14,7 +14,6 @@ from StringIO import StringIO
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
-from calibre.utils.logging import Log
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
@@ -109,9 +108,6 @@ def do_download_for_worker(book,options):
     when run as a worker job
     '''
     try:
-        # import logging
-        # logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
-        
         book['comment'] = 'Download started...'
 
         configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
@@ -123,13 +119,14 @@ def do_download_for_worker(book,options):
 
         # images only for epub, even if the user mistakenly turned it
         # on else where.
-        if options['fileform'] != "epub":
+        if options['fileform'] not in ("epub","html"):
             configuration.set("overrides","include_images","false")
         
         adapter = adapters.getAdapter(configuration,book['url'])
         adapter.is_adult = book['is_adult'] 
         adapter.username = book['username'] 
         adapter.password = book['password']
+        adapter.setChaptersRange(book['begin'],book['end'])
         
         story = adapter.getStoryMetadataOnly()
         if 'calibre_series' in book:
@@ -152,8 +149,6 @@ def do_download_for_worker(book,options):
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
 
-            adapter.setChaptersRange(book['begin'],book['end'])
-            
             print("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))

downloader.py

@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-import logging
-
 import sys, os
 from os.path import normpath, expanduser, isfile, join
 from StringIO import StringIO
@@ -26,6 +24,14 @@ import string
 import ConfigParser
 from subprocess import call
 
+import logging
+if sys.version_info >= (2, 7):
+    # suppresses default logger.  Logging is setup in fanficdownload/__init__.py so it works in calibre, too.
+    rootlogger = logging.getLogger()
+    loghandler=logging.NullHandler()
+    loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
+    rootlogger.addHandler(loghandler)
+
 from fanficdownloader import adapters,writers,exceptions
 from fanficdownloader.configurable import Configuration
 from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
@@ -79,11 +85,9 @@ def main():
    
    (options, args) = parser.parse_args()
 
-   if options.debug:
-       logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
-   else:
-       logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
-
+   if not options.debug:
+       logger = logging.getLogger("fanficdownloader")
+       logger.setLevel(logging.INFO)
    
    if len(args) != 1:
        parser.error("incorrect number of arguments")
@@ -120,8 +124,6 @@ def main():
    logging.debug('reading %s config file(s), if present'%conflist)
    configuration.read(conflist)
 
-   print("has include_in_tags?%s"%configuration.hasConfig("include_in_tags"))
-
    try:
        configuration.add_section("overrides")
    except ConfigParser.DuplicateSectionError:
@@ -138,7 +140,7 @@ def main():
 
    # images only for epub, even if the user mistakenly turned it
    # on else where.
-   if options.format != "epub":
+   if options.format not in ("epub","html"):
        configuration.set("overrides","include_images","false")
        
    if options.options:
@@ -156,6 +158,8 @@ def main():
 
        adapter = adapters.getAdapter(configuration,url)
 
+       adapter.setChaptersRange(options.begin,options.end)
+           
        ## Check for include_images and absence of PIL, give warning.
        if adapter.getConfig('include_images'):
            try:
@@ -219,8 +223,6 @@ def main():
            if options.metaonly:
                print adapter.getStoryMetadataOnly()
            
-           adapter.setChaptersRange(options.begin,options.end)
-           
            output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        
        if not options.metaonly and adapter.getConfig("post_process_cmd"):

fanficdownloader/__init__.py

@@ -1 +1,16 @@
 # -*- coding: utf-8 -*-
+
+try:
+    # just a way to switch between web service and CLI/PI
+    import google.appengine.api 
+except:
+    import sys
+    if sys.version_info >= (2, 7):
+        import logging
+        logger = logging.getLogger(__name__)
+        loghandler=logging.StreamHandler()
+        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+        logger.addHandler(loghandler)
+        loghandler.setLevel(logging.DEBUG)
+        logger.setLevel(logging.DEBUG)
+

fanficdownloader/adapters/__init__.py

@@ -1,176 +1,198 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob, types
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-from .. import exceptions as exceptions
-
-## must import each adapter here.
-
-import adapter_test1
-import adapter_fanfictionnet
-import adapter_castlefansorg
-import adapter_fictionalleyorg
-import adapter_fictionpresscom
-import adapter_ficwadcom
-import adapter_fimfictionnet
-import adapter_harrypotterfanfictioncom
-import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
-import adapter_tenhawkpresentscom
-import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
-import adapter_tthfanficorg
-import adapter_twilightednet
-import adapter_twiwritenet
-import adapter_whoficcom
-import adapter_siyecouk
-import adapter_archiveofourownorg
-import adapter_ficbooknet
-import adapter_portkeyorg
-import adapter_mugglenetcom
-import adapter_hpfandomnet
-import adapter_thequidditchpitchorg
-import adapter_nfacommunitycom
-import adapter_midnightwhispersca
-import adapter_ksarchivecom
-import adapter_archiveskyehawkecom
-import adapter_squidgeorgpeja
-import adapter_libraryofmoriacom
-import adapter_wraithbaitcom
-import adapter_checkmatedcom
-import adapter_chaossycophanthexcom
-import adapter_dramioneorg
-import adapter_erosnsapphosycophanthexcom
-import adapter_lumossycophanthexcom
-import adapter_occlumencysycophanthexcom
-import adapter_phoenixsongnet
-import adapter_walkingtheplankorg
-import adapter_ashwindersycophanthexcom
-import adapter_thehexfilesnet
-import adapter_dokugacom
-import adapter_iketernalnet
-import adapter_onedirectionfanfictioncom
-import adapter_prisonbreakficnet
-import adapter_storiesofardacom
-import adapter_yourfanfictioncom
-import adapter_samdeanarchivenu
-import adapter_destinysgatewaycom
-import adapter_ncisfictioncom
-import adapter_stargateatlantisorg
-import adapter_thealphagatecom
-import adapter_fanfiktionde
-import adapter_ponyfictionarchivenet
-import adapter_sg1heliopoliscom
-import adapter_ncisficcom
-import adapter_nationallibrarynet
-import adapter_themasquenet
-import adapter_pretendercentrecom
-import adapter_darksolaceorg
-import adapter_finestoriescom
-import adapter_hpfanficarchivecom
-import adapter_svufictioncom
-import adapter_twilightarchivescom
-import adapter_wizardtalesnet
-import adapter_nhamagicalworldsus
-import adapter_hlfictionnet
-import adapter_grangerenchantedcom
-import adapter_dracoandginnycom
-import adapter_scarvesandcoffeenet
-import adapter_thepetulantpoetesscom
-import adapter_wolverineandroguecom
-import adapter_sinfuldesireorg
-import adapter_merlinficdtwinscouk
-import adapter_thehookupzonenet
-import adapter_bloodtiesfancom
-
-## This bit of complexity allows adapters to be added by just adding
-## importing.  It eliminates the long if/else clauses we used to need
-## to pick out the adapter.
-    
-## List of registered site adapters.
-__class_list = []
-
-def imports():
-    for name, val in globals().items():
-        if isinstance(val, types.ModuleType):
-            yield val.__name__
-
-for x in imports():
-    if "fanficdownloader.adapters.adapter_" in x:
-        #print x
-        __class_list.append(sys.modules[x].getClass())
-
-def getAdapter(config,url):
-
-    logging.debug("trying url:"+url)
-    (cls,fixedurl) = getClassFor(url)
-    logging.debug("fixedurl:"+fixedurl)
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getConfigSections():
-    return [cls.getConfigSection() for cls in __class_list]
-
-def getConfigSectionFor(url):
-    (cls,fixedurl) = getClassFor(url)
-    if cls:
-        return cls.getConfigSection()
-    
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getClassFor(url):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    cls = getClassFromList(domain)
-    if not cls and domain.startswith("www."):
-        domain = domain.replace("www.","")
-        logging.debug("trying site:without www: "+domain)
-        cls = getClassFromList(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
-    if not cls:
-        logging.debug("trying site:www."+domain)
-        cls = getClassFromList("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-        
-    return (cls,fixedurl)
-    
-def getClassFromList(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, re, sys, glob, types
+from os.path import dirname, basename, normpath
+import logging
+import urlparse as up
+
+logger = logging.getLogger(__name__)
+
+from .. import exceptions as exceptions
+from ..configurable import Configuration
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
+import adapter_archiveofourownorg
+import adapter_ficbooknet
+import adapter_portkeyorg
+import adapter_mugglenetcom
+import adapter_hpfandomnet
+import adapter_thequidditchpitchorg
+import adapter_nfacommunitycom
+import adapter_midnightwhispersca
+import adapter_ksarchivecom
+import adapter_archiveskyehawkecom
+import adapter_squidgeorgpeja
+import adapter_libraryofmoriacom
+import adapter_wraithbaitcom
+import adapter_checkmatedcom
+import adapter_chaossycophanthexcom
+import adapter_dramioneorg
+import adapter_erosnsapphosycophanthexcom
+import adapter_lumossycophanthexcom
+import adapter_occlumencysycophanthexcom
+import adapter_phoenixsongnet
+import adapter_walkingtheplankorg
+import adapter_ashwindersycophanthexcom
+import adapter_thehexfilesnet
+import adapter_dokugacom
+import adapter_iketernalnet
+import adapter_onedirectionfanfictioncom
+import adapter_prisonbreakficnet
+import adapter_storiesofardacom
+import adapter_samdeanarchivenu
+import adapter_destinysgatewaycom
+import adapter_ncisfictionnet
+import adapter_stargateatlantisorg
+import adapter_thealphagatecom
+import adapter_fanfiktionde
+import adapter_ponyfictionarchivenet
+import adapter_sg1heliopoliscom
+import adapter_ncisficcom
+import adapter_nationallibrarynet
+import adapter_themasquenet
+import adapter_pretendercentrecom
+import adapter_darksolaceorg
+import adapter_finestoriescom
+import adapter_hpfanficarchivecom
+import adapter_svufictioncom
+import adapter_twilightarchivescom
+import adapter_wizardtalesnet
+import adapter_nhamagicalworldsus
+import adapter_hlfictionnet
+import adapter_grangerenchantedcom
+import adapter_dracoandginnycom
+import adapter_scarvesandcoffeenet
+import adapter_thepetulantpoetesscom
+import adapter_wolverineandroguecom
+import adapter_sinfuldesireorg
+import adapter_merlinficdtwinscouk
+import adapter_thehookupzonenet
+import adapter_bloodtiesfancom
+import adapter_indeathnet
+import adapter_jlaunlimitedcom
+import adapter_qafficcom
+import adapter_efpfanficnet
+#import adapter_buffynfaithnet
+import adapter_potterficscom
+
+## This bit of complexity allows adapters to be added by just adding
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
+    
+## List of registered site adapters.
+__class_list = []
+
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        __class_list.append(sys.modules[x].getClass())
+
+def getNormalStoryURL(url):
+    if not getNormalStoryURL.__dummyconfig:
+        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
+    # pulling up an adapter is pretty low over-head.  If
+    # it fails, it's a bad url.
+    try:
+        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
+        url = adapter.url
+        del adapter
+        return url
+    except:
+        return None;
+
+# kludgey function static/singleton
+getNormalStoryURL.__dummyconfig = None
+
+def getAdapter(config,url):
+
+    logger.debug("trying url:"+url)
+    (cls,fixedurl) = getClassFor(url)
+    logger.debug("fixedurl:"+fixedurl)
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getConfigSections():
+    return [cls.getConfigSection() for cls in __class_list]
+
+def getConfigSectionFor(url):
+    (cls,fixedurl) = getClassFor(url)
+    if cls:
+        return cls.getConfigSection()
+    
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getClassFor(url):
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+    
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    cls = getClassFromList(domain)
+    if not cls and domain.startswith("www."):
+        domain = domain.replace("www.","")
+        logger.debug("trying site:without www: "+domain)
+        cls = getClassFromList(domain)
+        fixedurl = fixedurl.replace("http://www.","http://")
+    if not cls:
+        logger.debug("trying site:www."+domain)
+        cls = getClassFromList("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+        
+    fixedurl = cls.stripURLParameters(fixedurl)
+    
+    return (cls,fixedurl)
+    
+def getClassFromList(domain):
+    for cls in __class_list:
+        if cls.matchesSite(domain):
+            return cls

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -41,7 +42,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,7 +66,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -204,7 +205,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -30,6 +31,8 @@ def getClass():
     return ArchiveOfOurOwnOrgAdapter
 
 
+logger = logging.getLogger(__name__)
+
 class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -48,13 +51,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
         else:
@@ -104,14 +107,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         params['authenticity_token'] = data.split('input name="authenticity_token" type="hidden" value="')[1].split('" /></div>')[0]
 
         loginUrl = 'http://' + self.getSiteDomain() + '/user_sessions'
-        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['user_session[login]']))
     
         d = self._postUrl(loginUrl, params)
-        #logging.info(d)
+        #logger.info(d)
     
         if "Successfully logged in" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['user_session[login]']))
             raise exceptions.FailedToLogin(url,params['user_session[login]'])
             return False
@@ -128,8 +131,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
         metaurl = self.url+addurl
         url = self.url+'/navigate'+addurl
-        logging.info("url: "+url)
-        logging.info("metaurl: "+metaurl)
+        logger.info("url: "+url)
+        logger.info("metaurl: "+metaurl)
 
         try:
             data = self._fetchUrl(url)
@@ -164,7 +167,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
         if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
             self.story.setMetadata('author','Anonymous')
-            self.story.setMetadata('authorUrl',self.url)
+            self.story.setMetadata('authorUrl','http://archiveofourown.org/')
+            self.story.setMetadata('authorId','0')
         else:
             for a in alist:
                 self.story.addToList('authorId',a['href'].split('/')[2])
@@ -174,7 +178,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
         self.story.setMetadata('numChapters',len(chapters))
-        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
         for x in range(0,len(chapters)):
             # just in case there's tags, like <i> in chapter titles.
             chapter=chapters[x]
@@ -213,7 +217,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
-                self.story.addToList('freefromtags',genre.string)
+                self.story.addToList('freeformtags',genre.string)
                 self.story.addToList('genre',genre.string)
                 
         a = metasoup.find('dd',{'class':"category tags"})
@@ -229,13 +233,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chars = a.findAll('a',{'class':"tag"})
             for char in chars:
                 self.story.addToList('characters',char.string)
+                
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
             ships = a.findAll('a',{'class':"tag"})
             for ship in ships:
                 self.story.addToList('ships',ship.string)
 		
-
+        a = metasoup.find('dd',{'class':"collections"})
+        if a != None:
+            collections = a.findAll('a')
+            for collection in collections:
+                self.story.addToList('collections',collection.string)
+		
         stats = metasoup.find('dl',{'class':'stats'})
         dt = stats.findAll('dt')
         dd = stats.findAll('dd')
@@ -291,7 +301,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         chapter=bs.BeautifulSoup('<div class="story"></div>')
         data = self._fetchUrl(url)

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
@@ -78,7 +79,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -151,7 +152,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
                 rating.find('br').replaceWith('split')
                 rating=rating.text.replace("This story is rated",'').split('split')[0]
                 self.story.setMetadata('rating',rating)
-                logging.debug(self.story.getMetadata('rating'))
+                logger.debug(self.story.getMetadata('rating'))
 			
                 warnings=box.find('ol')
                 if warnings != None:
@@ -177,7 +178,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -94,13 +95,13 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Logout" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -112,7 +113,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -237,7 +238,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
 

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -69,7 +70,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -115,13 +116,13 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -148,7 +149,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -182,7 +183,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -320,7 +321,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -71,7 +72,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -117,13 +118,13 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -145,7 +146,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -293,7 +294,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -86,7 +87,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -221,7 +222,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
 
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 
         self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
 
@@ -95,7 +96,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         e = self._fetchUrl(url)
 
         if "Welcome back," not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['name']))
             raise exceptions.FailedToLogin(url,params['name'])
             return False
@@ -112,7 +113,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -222,7 +223,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -98,13 +99,13 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "User Account Page" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
+            logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
         else:
@@ -115,7 +116,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -261,7 +262,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -86,7 +87,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -106,7 +107,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -227,7 +228,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_dokugacom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 
         # www.dokuga.com has two 'sections', shown in URL as
         # 'fanfiction' and 'spark' that change how things should be
@@ -87,7 +88,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -224,7 +225,7 @@ class DokugaComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -92,13 +93,13 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -120,7 +121,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -145,7 +146,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -282,7 +283,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -92,13 +93,13 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -120,7 +121,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -283,7 +284,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -0,0 +1,312 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return EFPFanFicNet
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class EFPFanFicNet(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','efp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.efpfanfic.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Fai il login e leggi la storia!' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Invia'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?sid='+self.story.getMetadata('storyId')
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if '<a class="menu" href="newaccount.php">' in d : # register for new account link
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'sid' } )
+    	 
+        if select is None:
+    	   # no selector found, so it's a one-chapter story.
+    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
+            for o in allOptions:
+                url = u'http://%s/%s' % ( self.getSiteDomain(),
+                                          o['value'])
+                # just in case there's tags, like <i> in chapter titles.
+                title = stripHTML(o)
+                self.chapterUrls.append((title,url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','Italian')
+        
+        # normalize story URL to first chapter if later chapter URL was given:
+        url = self.chapterUrls[0][1].replace('&i=1','')
+        logger.debug("Normalizing to URL: "+url)
+        self._setURL(url)
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        storya = None
+        authsoup = None
+        storyblock = None
+        authurl = self.story.getMetadata('authorUrl')
+        
+        ## author can have more than one page of stories.
+        while storyblock == None:
+
+            # no storya, but do have authsoup--we're looping on author pages.
+            if authsoup != None:
+                # last author link with offset should be the 'next' link.
+                authurl = u'http://%s/%s' % ( self.getSiteDomain(),
+                                              authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
+            
+            # Need author page for most of the metadata.
+            logger.debug("fetching author page: (%s)"%authurl)
+            authsoup = bs.BeautifulSoup(self._fetchUrl(authurl))
+            #print("authsoup:%s"%authsoup)
+
+            storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
+            for storya in storyas:
+                #print("======storya:%s"%storya)
+                storyblock = storya.findParent('div',{'class':'storybloc'})
+                #print("======storyblock:%s"%storyblock)
+                if storyblock != None:
+                    continue
+
+        self.setDescription(url,storyblock.find('div', {'class':'introbloc'}))
+
+        noteblock = storyblock.find('div', {'class':'notebloc'})
+        #print("%s"%noteblock)
+        notetext = ("%s" % noteblock).replace("<br />"," |")
+        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />␍
+        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />␍
+        # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>
+
+        cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        for item in notetext.split("|"):
+            if ":" in item:
+                (label,value) = item.split(":")
+                label=label.strip()
+                value=value.strip()
+            else:
+                label=value=item.strip()
+
+            if 'Pubblicata' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Aggiornata' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            if label == "Completa":
+                self.story.setMetadata('status', 'Completed')
+            
+            if label == "In corso":
+                self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Personaggi' in label:
+                for val in value.split(","):
+                    self.story.addToList('characters',val)
+                    
+            if 'Genere' in label:
+                for val in value.split(","):
+                    self.story.addToList('genre',val)
+                    
+            if 'Coppie' in label:
+                for val in value.split(","):
+                    self.story.addToList('ships',val)
+
+            if 'Avvertimenti' in label:
+                for val in value.split(","):
+                    if val != "None":
+                        self.story.addToList('warnings',val)
+
+            # 'extra' metadata for this adapter:
+                    
+            if 'Tipo di coppia' in label:
+                for val in value.split(","):
+                    self.story.addToList('type',val)
+                    
+            if 'Note' in label:
+                for val in value.split(","):
+                    if val != "None":
+                        self.story.addToList('notes',val)
+                    
+            if 'Contesto' in label:
+                self.story.setMetadata('context', value)
+
+            ## Note--efp doesn't provide word count.
+                
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?ssid=\d+&i=1"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'class' : 'storia'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        # remove any header and 'o:p' tags.
+        for tag in div.findAll("head") + div.findAll("o:p"):
+            tag.extract()
+
+        # change any html and body tags to div.
+        for tag in div.findAll("html") + div.findAll("body"):
+            tag.name='div'
+
+        # remove extra bogus doctype.
+        #<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+        return re.sub(r"<!DOCTYPE[^>]+>","",self.utf8FromSoup(url,div))

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -86,7 +87,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -106,7 +107,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -239,7 +240,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 from urllib import unquote_plus
@@ -67,7 +68,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -75,12 +76,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # metadata and chapter list
 
         url = self.origurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
-            #print("\n===================\n%s\n===================\n"%data)
+            #logger.debug("\n===================\n%s\n===================\n"%data)
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -108,11 +109,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
                                            self.story.getMetadata('storyId'),
                                            chapcount+1)
-            print('=Trying newer chapter: %s' % tryurl)
+            logger.debug('=Trying newer chapter: %s' % tryurl)
             newdata = self._fetchUrl(tryurl)
             if "not found. Please check to see you are not using an outdated url." \
                     not in newdata:
-                print('=======Found newer chapter: %s' % tryurl)
+                logger.debug('=======Found newer chapter: %s' % tryurl)
                 soup = bs.BeautifulSoup(newdata)
         except:
             pass
@@ -160,7 +161,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         
         metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
         metalist = metatext.split(" - ")
-        #print("metatext:(%s)"%metalist)
+        #logger.debug("metatext:(%s)"%metalist)
 
         # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
 
@@ -176,9 +177,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
         goodgenres=True
         for g in genrelist:
-            print("g:(%s)"%g)
+            #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                print("g not in ffnetgenres")
+                logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -214,9 +215,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        img = soup.find('img',{'class':'cimage'})
-        if img:
-            self.setCoverImage(url,img['src'])
+        # Try the larger image first.
+        try:
+            img = soup.find('img',{'class':'lazy cimage'})
+            self.setCoverImage(url,img['data-original'])
+        except:
+            img = soup.find('img',{'class':'cimage'})
+            if img:
+                self.setCoverImage(url,img['src'])
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapter' } )
@@ -240,7 +246,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
@@ -265,7 +271,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         div = soup.find('div', {'id' : 'storytextp'})
         
         if None == div:
-            logging.debug('div id=storytextp not found.  data:%s'%data)
+            logger.debug('div id=storytextp not found.  data:%s'%data)
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -25,6 +26,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
+
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
@@ -48,7 +50,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
@@ -94,12 +96,12 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         params['submit'] = 'Login...'
 
         loginUrl = 'https://ssl.fanfiktion.de/'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['nickname']))
         d = self._postUrl(loginUrl,params)
     
         if "Login erfolgreich" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['nickname']))
             raise exceptions.FailedToLogin(url,params['nickname'])
             return False
@@ -110,7 +112,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -139,7 +141,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        head = soup.find('div', {'style' : 'width:85%;float:left;'})
+        head = soup.find('div', {'class' : 'story-metadata-left-top'})
         a = head.find('a')
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
@@ -175,7 +177,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         self.setDescription(url,a['onmouseover'].split("', '")[1])
         
         td = tr[i].findAll('td')
-        self.story.addToList('category',td[1].string)
+        self.story.addToList('category',stripHTML(td[1]))
         self.story.setMetadata('rating', td[4].string)
         self.story.setMetadata('numWords', td[5].string)
  
@@ -183,7 +185,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffde has "floodlock" protection
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -18,6 +18,7 @@
 import time
 import datetime
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 from .. import translit
@@ -33,6 +34,8 @@ def getClass():
     return FicBookNetAdapter
 
 
+logger = logging.getLogger(__name__)
+
 class FicBookNetAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -49,7 +52,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
@@ -75,7 +78,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
         url=self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
         try:
             data = self._fetchUrl(url)
         except urllib2.HTTPError, e:
@@ -95,14 +98,14 @@ class FicBookNetAdapter(BaseSiteAdapter):
         ## Title
         a = soup.find('h1')
         self.story.setMetadata('title',stripHTML(a))
-        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
+        logger.debug("Title: (%s)"%self.story.getMetadata('title'))
         
         # Find authorid and URL from... author url.
         a = table.find('a')
         self.story.setMetadata('authorId',a.text) # Author's name is unique
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.text)
-        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
+        logger.debug("Author: (%s)"%self.story.getMetadata('author'))
 
         # Find the chapters:
         chapters = soup.find('div', {'class' : 'part_list'})
@@ -123,7 +126,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
             pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
             update=pubdate
 
-        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
 
         if not ',' in pubdate:
             pubdate=datetime.date.today().strftime(self.dateformat)
@@ -207,7 +210,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -44,7 +45,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if m:
             self.story.setMetadata('authorId',m.group('auth'))
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL(url)
         else:
@@ -68,7 +69,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if self.is_adult or self.getConfig("is_adult"):
             params={'iamold':'Yes',
                     'action':'ageanswer'}
-            logging.info("Attempting to get cookie for %s" % url)
+            logger.info("Attempting to get cookie for %s" % url)
             ## posting on list doesn't work, but doesn't hurt, either.
             data = self._postUrl(url,params)
         else:
@@ -79,7 +80,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
         ## could be either chapter list page or one-shot text page.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._postFetchWithIAmOld(url)
@@ -107,7 +108,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             storya = soup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
             self._setURL(storya['href'])
             url=self.url
-            logging.debug("Normalizing to URL: "+url)
+            logger.debug("Normalizing to URL: "+url)
             ## title's right there...
             self.story.setMetadata('title',storya.string)
             data = self._fetchUrl(url)
@@ -193,7 +194,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
 	# find <!-- headerend --> & <!-- footerstart --> and
@@ -222,7 +223,16 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             
         if not data or not text:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
+        # not sure how, but we can get html, etc tags still in some
+        # stories.  That breaks later updates because it confuses
+        # epubutils.py
+        for tag in text.findAll('head'):
+            tag.extract()
+        
+        for tag in text.findAll('body') + text.findAll('html'):
+            tag.name = 'div'
+            
         return self.utf8FromSoup(url,text)
 
 def getClass():

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time
@@ -61,12 +62,12 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             params['password'] = self.getConfig("password")
 
         loginUrl = 'http://' + self.getSiteDomain() + '/account/login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['username']))
         d = self._postUrl(loginUrl,params)
 
         if "Login attempt failed..." in d:
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['username']))
             raise exceptions.FailedToLogin(url,params['username'])
             return False
@@ -79,7 +80,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         # metadata and chapter list
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
@@ -96,7 +97,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             # normalize story URL on chapter list.
             self.story.setMetadata('storyId',storya['href'].split('/',)[2])
             url = "http://"+self.getSiteDomain()+storya['href']
-            logging.debug("Normalizing to URL: "+url)
+            logger.debug("Normalizing to URL: "+url)
             self._setURL(url)
             try:
                 soup = bs.BeautifulSoup(self._fetchUrl(url))
@@ -201,7 +202,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
@@ -119,15 +120,23 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
         
         chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
+        
+        ## this is bit of a kludge based on the assumption all the
+        ## 'bad' chapters will be at the end.
+        ## limit down to the number of chapters reported by chapter_count.
+        chapters = chapters[:storyMetadata["chapter_count"]] 
+        
         for chapter in chapters:
             self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
         self.story.setMetadata("numChapters", len(self.chapterUrls))
         
         # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
-        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
+        # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
         # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
         # status to leave unchanged.
-        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
+        # Nov2012 - 'On Hiatus' is now passed, too.  It's easy now for users to change/remove if they want
+        # with replace_metadata
+        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata("status", status)
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
             
@@ -143,8 +152,9 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 coverurl = storyMetadata["full_image"]
             else:
                 coverurl = storyMetadata["image"]
-            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
+            if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
+
             self.setCoverImage(self.url,coverurl)
 
         # the fimfic API gives bbcode for desc, not html.
@@ -158,7 +168,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
         rawDateUpdated = storyMetadata["date_modified"]
         self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
-        
+
         soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
@@ -173,9 +183,18 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 char = "Cutie Mark Crusaders"
             self.story.addToList("characters", char)
             
+        # extra site specific metadata
+        extralist = ["likes","dislikes","views","total_views","short_description"]
+        for metakey in extralist:
+            if metakey in storyMetadata:
+                value = storyMetadata[metakey]
+                if not isinstance(value,basestring):
+                    value = unicode(value)
+                self.story.setMetadata(metakey, value)
+        
             
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -49,7 +50,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
@@ -95,13 +96,13 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         params['submit'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['theusername']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "My Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['theusername']))
             raise exceptions.FailedToLogin(url,params['theusername'])
             return False
@@ -114,7 +115,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -232,7 +233,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,8 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.section=self.parsedUrl.path.split('/',)[1]
              
         # normalized story URL.
         if "malfoymanor" in self.parsedUrl.netloc:
@@ -100,17 +101,17 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         params['cookiecheck'] = '1'
         params['submit'] = 'Submit'
     
-        if "enchant" in self.story.getMetadata('section'):
+        if "enchant" in self.section:
             loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
         else:
             loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -132,7 +133,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -157,7 +158,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -191,7 +192,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -262,7 +263,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -284,10 +285,10 @@ class GrangerEnchantedCom(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'id' : 'story1'})
 

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -41,7 +42,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only psid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
@@ -72,7 +73,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -181,7 +182,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         ## most adapters use BeautifulStoneSoup here, but non-Stone
         ## allows nested div tags.

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -76,7 +77,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -215,7 +216,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
@@ -79,7 +80,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -180,21 +181,22 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
             value = td.nextSibling.string
             #print("\nlabel:%s\nvalue:%s\n"%(label,value))
 
-            if 'Category' in label:
+            if 'Category' in label and value:
                 cats = td.parent.findAll('a',href=re.compile(r'categories.php'))
                 catstext = [cat.string for cat in cats]
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
-            if 'Characters' in label:
+            if 'Characters' in label and value: # this site can have Character label with no
+                                                # values, apparently.  Others as a precaution.
                 for char in value.split(','):
                     self.story.addToList('characters',char.strip())
 
-            if 'Genre' in label:
+            if 'Genre' in label and value:
                 for genre in value.split(','):
                     self.story.addToList('genre',genre.strip())
 
-            if 'Warnings' in label:
+            if 'Warnings' in label and value:
                 for warning in value.split(','):
                     if warning.strip() != 'none':
                         self.story.addToList('warnings',warning.strip())
@@ -208,7 +210,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # There's no good wrapper around the chapter text. :-/

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -76,7 +77,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -202,7 +203,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -92,13 +93,13 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -120,7 +121,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -158,7 +159,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -267,7 +268,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_indeathnet.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return InDeathNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class InDeathNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+		
+		  # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/')
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','idn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'indeath.net'
+
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/"
+
+    def getSiteURLPattern(self):
+        # http://www.indeath.net/blog/archive/169-ransom-in-death/
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"/blog/(archive/)?(?P<id>\d+)\-(?P<name>[a-z0-9\-]*)/?$"     
+        
+            
+    def getDateFromComponents(self, postmonth, postday):
+        ym = re.search("Entries\ in\ (?P<mon>January|February|March|April|May|June|July|August|September|October|November|December)\ (?P<year>\d{4})",postmonth)
+        d = re.search("(?P<day>\d{2})\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)",postday)
+        postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
+        return postdate
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        try:
+            data = self._fetchUrl(url)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+                
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        h = soup.find('a', id="blog_title")
+        t = h.find('span')
+        self.story.setMetadata('title',t.contents[0].string.strip())
+		
+        s = t.find('div')        
+        if s != None:
+            self.setDescription(url,s)
+		
+        # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required)
+        a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$"))		#http://www.indeath.net/user/9083-cyrex/
+        m = re.search('http://www.indeath.net/user/(?P<id>\d+)\-(?P<name>[a-z0-9]*)/$',a['href'])
+        self.story.setMetadata('authorId',m.group('id'))
+        self.story.setMetadata('authorUrl',a['href'])
+        self.story.setMetadata('author',m.group('name'))
+
+
+
+
+        # Find the chapters:
+        chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
+
+        #reverse the list since newest at the top
+        chapters.reverse()
+        
+        # Get date published & updated from first & last entries
+        posttable=soup.find('div', id="main_column")
+        
+        postmonths=posttable.findAll('th', text=re.compile(r'Entries\ in\ '))
+        postmonths.reverse()
+        
+        postdates=posttable.findAll('span', _class="desc", text=re.compile('\d{2}\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'))
+        postdates.reverse()
+        
+        self.story.setMetadata('datePublished',self.getDateFromComponents(postmonths[0],postdates[0]))
+        self.story.setMetadata('dateUpdated',self.getDateFromComponents(postmonths[len(postmonths)-1],postdates[len(postdates)-1]))
+        
+        # Process List of Chapters              
+        self.story.setMetadata('numChapters',len(chapters))
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        for x in range(0,len(chapters)):
+            # just in case there's tags, like <i> in chapter titles.
+            chapter=chapters[x]
+            if len(chapters)==1:
+                self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
+            else:
+            	ct = stripHTML(chapter)
+            	tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+            	if tnew:
+            		chaptertitle = tnew.group('newtitle')
+            	else:
+            		chaptertitle = ct
+                self.chapterUrls.append((chaptertitle,chapter['href']))
+	
+		
+        
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+		
+        #chapter=bs.BeautifulSoup('<div class="story"></div>')
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))
+
+        chapter = soup.find("div", "entry_content")
+        
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chapter)
+

fanficdownloader/adapters/adapter_jlaunlimitedcom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2011 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -26,56 +27,51 @@ from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter,  makeDate
 
-def getClass():
-    return YourFanfictionComAdapter
 
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class YourFanfictionComAdapter(BaseSiteAdapter):
+def getClass():
+    return JLAUnlimitedComAdapter
+
+
+class JLAUnlimitedComAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # yourfanfiction.com blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted as us and offered to 'whitelist our IP'.  Clearly,
-        # that wouldn't work, but it does let me do this in good
-        # conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
-
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.username = "" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-        
+
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        self._setURL('http://' + self.getSiteDomain() + '/eFiction1.1/viewstory.php?sid='+self.story.getMetadata('storyId'))
+
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','yff')
+        self.story.setMetadata('siteabbrev','jla')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-            
+        self.dateformat = "%m/%d/%y"
+
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.yourfanfiction.com'
+        return 'www.jlaunlimited.com'
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=")+r"\d+$"
+
+
+
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -85,14 +81,14 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
+            addurl = "&ageconsent=ok&warning=4" # XXX
         else:
             addurl=""
 
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -102,20 +98,8 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((&ageconsent=ok)?&warning=\d+)'",data)
+        # Assume that if there is a url with 'warning=#' in the page then it is a 'check' page
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -123,10 +107,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 # link and reload data.
                 addurl = m.group(1)
                 # correct stupid & error in url.
-                # explicitly put ageconsent because google appengine regexp doesn't include it for some reason.
-                addurl = addurl.replace("&","&")+'&ageconsent=ok'
+                addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -137,26 +120,15 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                         raise e    
             else:
                 raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # because for some reason, this works while simple 'print data' errors on ascii conversion.
-        # loopdata = data
-        # chklen=5000
-        # while len(loopdata) > 0:
-        #     if len(loopdata) < 5000:
-        #         chklen = len(loopdata)
-        #     logging.info("loopdata: %s" % loopdata[:chklen])
-        #     loopdata = loopdata[chklen:]
-            
+                
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
+        # print data
 
         # Now go hunting for all the meta data and the chapter list.
-        
+
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
@@ -166,9 +138,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eFiction1.1/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -191,11 +163,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while value and not defaultGetattr(value,'class') == 'label':
+                while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                # sometimes poorly formated desc (<p> w/o </p>) leads
-                # to all labels being included.
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -207,27 +177,34 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
+                charstext = [char.string for char in chars]
+                for char in charstext:
                     self.story.addToList('characters',char.string)
 
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-                    
-            if 'Tags' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=7'))
-                for genre in genres:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
                     self.story.addToList('genre',genre.string)
 
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
-                for warning in warnings:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
@@ -248,36 +225,35 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
+            series_url = 'http://'+self.host+'/fanfic/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
             
         except:
             # I find it hard to care if the series parsing fails
             pass
-            
+
+
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
 
+        div = soup.find('div', {'id' : 'story'})
+        
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
         return self.utf8FromSoup(url,div)
+

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -99,7 +100,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -132,7 +133,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -294,7 +295,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -84,7 +85,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -104,7 +105,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -234,7 +235,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -86,7 +87,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -221,7 +222,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -42,7 +43,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
         else:
@@ -66,7 +67,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -188,7 +189,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data=self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,
@@ -214,7 +215,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
             return self.utf8FromSoup(url,anchor)
         
         else:
-            logging.debug('Using kludgey text find for older mediaminer story.')
+            logger.debug('Using kludgey text find for older mediaminer story.')
             ## Some older mediaminer stories are unparsable with BeautifulSoup.
             ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
             ## Story stuff falls between:

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -92,13 +93,13 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -120,7 +121,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -145,7 +146,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -277,7 +278,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -95,7 +96,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -128,7 +129,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -269,7 +270,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -96,13 +97,13 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId')
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -125,7 +126,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -164,7 +165,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -315,7 +316,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -80,7 +81,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -177,7 +178,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -80,7 +81,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -184,7 +185,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -27,15 +28,20 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return NCISFictionComAdapter
+    return NCISFictionNetAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class NCISFictionComAdapter(BaseSiteAdapter):
+class NCISFictionNetAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
+        # ncisfiction.net blocks IPs the default user-agent.  However,
+        # when asked, they said it was just general anti-spam, not
+        # targeted at us.  That lets me do this in good conscience:
+        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
+        
         self.decode = ["iso-8859-1",
                        "Windows-1252"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -47,7 +53,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL("http://"+self.getSiteDomain()\
@@ -63,13 +69,21 @@ class NCISFictionComAdapter(BaseSiteAdapter):
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.ncisfiction.com'
+        return 'www.ncisfiction.net'
+
+    ## Changed from www.ncisfiction.com to www.ncisfiction.net Oct
+    ## 2012 due to the ncisfiction.com domain expiring.  Still accept
+    ## .com domains for existing updates, etc.
+    
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ncisfiction.net','www.ncisfiction.com']
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
 
     def getSiteURLPattern(self):
-        return "http://"+self.getSiteDomain()+r'/(chapters|story)?.php\?stid=\d+'
+        return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+'
        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
@@ -78,7 +92,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -187,7 +201,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -99,7 +100,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -132,7 +133,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -273,7 +274,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -74,7 +75,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -197,7 +198,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
 

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
 		
         
         # normalized story URL.
@@ -94,13 +95,13 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Logout" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -112,7 +113,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -245,7 +246,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
         data = data.replace('<div align="left"', '<div align="left">')

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -97,13 +98,13 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -125,7 +126,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -253,7 +254,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2, urllib, cookielib
 
@@ -47,7 +48,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
@@ -90,12 +91,12 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         params['login'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/users/processlogin.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['txtusername']))
         d = self._fetchUrl(loginUrl, params)
     
         if 'Please login to continue.' in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['txtusername']))
             raise exceptions.FailedToLogin(url,params['txtusername'])
             return False
@@ -108,7 +109,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -208,7 +209,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
 	# grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        logging.info(self.parsedUrl.netloc)
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.info(self.parsedUrl.netloc)
         # normalized story URL.
         if "explicit" in self.parsedUrl.netloc:
             self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -91,7 +92,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -112,7 +113,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -234,10 +235,9 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
@@ -54,7 +55,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
@@ -81,7 +82,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # portkey screws around with using a different URL to set the
         # cookie and it's a pain.  So... cheat!
@@ -247,7 +248,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -40,7 +41,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -63,7 +64,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -161,12 +162,12 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%b %d %Y"))
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
             
             if 'Updated' in label:
                 # there's a stray [ at the end.
                 #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.
@@ -191,7 +192,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_potterficscom.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return PotterFicsComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterFicsComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL. gets rid of chapter if there, left with chapter index URL
+            nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','potficscom')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.potterfics.com'
+
+    def getSiteExampleURLs(self):
+        return \
+                "http://www.potterfics.com/historias/127583 "\
+                "http://www.potterfics.com/historias/127583/capitulo-1 "\
+                "http://www.potterfics.com/historias/127583/capitulo-4 "\
+                "http://www.potterfics.com/historias/92810 "\
+                "http://www.potterfics.com/historias/111194"
+
+    def getSiteURLPattern(self):
+        #http://www.potterfics.com/historias/127583
+        #http://www.potterfics.com/historias/127583/capitulo-1
+        #http://www.potterfics.com/historias/127583/capitulo-4
+        #http://www.potterfics.com/historias/92810              -> Complete story
+        #http://www.potterfics.com/historias/111194             -> Complete, single chap
+        p = re.escape("http://"+self.getSiteDomain()+"/historias/")+\
+            r"(?P<id>\d+)(/capitulo-(?P<ch>\d+))?/?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        #this converts '/historias/12345' to 'http://www.potterfics.com/historias/12345'
+        def makeAbsoluteURL(url):
+            if url[0] == '/':
+                url = 'http://'+self.getSiteDomain()+url
+            return url
+
+        #use this to get month numbers from Spanish months
+        SpanishMonths = {
+            'enero'      : '01',
+            'febrero'    : '02',
+            'marzo'      : '03',
+            'abril'      : '04',
+            'mayo'       : '05',
+            'junio'      : '06',
+            'julio'      : '07',
+            'agosto'     : '08',
+            'septiembre' : '09',
+            'octubre'    : '10',
+            'noviembre'  : '11',
+            'diciembre'  : '12'
+            }
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        #deal with adult content warnings - doesn't seem to apply to this site
+
+        #set constant meta for this site:
+        #Set Language = Spanish
+        self.story.setMetadata('language', 'Spanish')
+        #Set Category = Harry Potter
+        #  This is better done in plugin-defaults.ini and defaults.ini
+        #  by adding a section for this site with the line:
+        #  extracategories:Harry Potter
+        #self.story.addToList('category','Harry Potter')
+        
+        #get the rest of the meta
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        #self closing br and img present!
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','img'))
+    
+        #we want the second table directly under the body, contains all the metadata
+        table = soup.html.body.findAll('table', recursive=False)[1]
+        #within that, we want the second row, first cell
+        cell = table.tr.findNextSibling('tr').td
+
+        #find first metadata block
+        mb = cell.div.findNextSibling('div')
+        #Get meta...
+        self.story.setMetadata('title', mb.b.string)
+        #strip out brackets on rating
+        self.story.setMetadata('rating', mb.span.string[1:-1])
+        #Completion status is denoted by the presence of this image:
+        if mb.find('img',title="Historia terminada"):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        #find next metadata block
+        #author details
+        mb = mb.findNextSibling('div')
+        self.story.setMetadata('author', mb.b.a.string.strip())
+        self.story.setMetadata('authorUrl', makeAbsoluteURL(mb.b.a['href']))
+        self.story.setMetadata('authorId', self.story.getMetadata('authorUrl').split('/')[4])
+        #dates and times
+        mb = mb.find('span')
+        #posted/published = Escrita
+        date = mb.find(text=re.compile('Escrita el ')).strip().split()
+        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
+        month = int(SpanishMonths[date[5].lower()])
+        day = int(date[3])
+        time = date[8].split(':')
+        hour = int(time[0])
+        minute = int(time[1])
+        self.story.setMetadata('datePublished', datetime.datetime(year, month, day, hour, minute))
+        #updated = Actualizada
+        date = mb.find(text=re.compile('Actualizada el ')).strip().split()
+        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
+        month = int(SpanishMonths[date[5].lower()])
+        day = int(date[3])
+        time = date[8].split(':')
+        hour = int(time[0])
+        minute = int(time[1])
+        self.story.setMetadata('dateUpdated', datetime.datetime(year, month, day, hour, minute))
+        
+        mb = mb.span.findNextSibling('span').findNextSibling('span')
+        wc = mb.find(text=re.compile(' palabras en total')).strip()
+        self.story.setMetadata('numWords', wc.split()[0])
+        
+        #then we come to categories and genres. Oh dear. On this site, categories hold everything from genre, to ships, to crossovers.
+        #To make things worse, there is also another genre field, which often holds similar/duplicate info. Links to genre pages do not work
+        #though, so perhaps those will be phased out?
+        #for now, put them all into the genre list
+        links = mb.findAll('a',href=re.compile('/(categorias|generos)/\d+'))
+        genlist = [i.string.strip() for i in links]
+        self.story.extendList('genre',genlist)
+        
+        #get the chapter urls
+        #we can go back to the table cell we found before
+        #get its last element and work backwards to find the last ordered list on the page
+        list = cell.contents[len(cell)-1].findPrevious('ol')
+        chapters = []
+        revs = 0
+        chnum = 0
+        for li in list:
+            chnum += 1
+            chTitle = str(chnum) + '. ' + li.a.b.string.strip()
+            chURL = makeAbsoluteURL(li.a['href'])
+            chapters.append((chTitle,chURL))
+            #Get reviews, add to total
+            revs += int(li.div.a.string.split()[0])
+        
+        self.chapterUrls.extend(chapters)
+        self.story.setMetadata('numChapters', len(chapters))
+        self.story.setMetadata('reviews', revs)
+        
+        #Now for the description... this may be tricky...
+        #if it is there (doesn't have to be), it will be before the chapter list,
+        #separated by a horizontal rule, and after the google ad bar
+        
+        #get list's parent div
+        mb = list.parent
+        #get the div before that, will either be the description, or the google ad bar
+        mb = mb.findPreviousSibling('div')
+        if 'google_ad_client' in str(mb):
+            #couldn't find description, leaving it blank
+            pass
+        else:
+            self.setDescription(url,mb)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr','img'))
+        
+        div = soup.find('div', {'id' : 'cuerpoHistoria'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -89,7 +90,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -109,7 +110,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -237,7 +238,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_prisonbreakficnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -76,7 +77,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -202,7 +203,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_qafficcom.py

@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return QafFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class QafFicComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','atp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.qaf-fic.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=")+r"\d+$"
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=NC-17"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\s+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/atp/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. 
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification' and value != None:
+                    if "Featured Stories" not in value:
+                        svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            if list.find('a', href=re.compile(r"series.php")) != None:
+                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
+                    # Find Series name from series URL.
+                    series_url = 'http://'+self.host+'/atp/'+series['href']
+                    # use BeautifulSoup HTML parser to make everything easier to find.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+                    i=1
+                    for a in storyas:
+                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                            name=seriessoup.find('div', {'id' : 'pagetitle'})
+                            name.find('a').extract()
+                            self.setSeries(name.text.split(' by[')[0], i)
+                            i=0
+                            break
+                        i+=1
+                    if i == 0:
+                        break
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -80,7 +81,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -216,7 +217,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -94,7 +95,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -114,7 +115,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -231,7 +232,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,11 +48,11 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.section=self.parsedUrl.path.split('/',)[1]
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/'+self.story.getMetadata('section')+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','sghp')
@@ -59,7 +60,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # If all stories from the site fall into the same category,
         # the site itself isn't likely to label them as such, so we
         # do.  Can't use extracategories, could be Atlantis or SG-1
-        if 'atlantis' in self.story.getMetadata('section'):
+        if 'atlantis' in self.section:
             self.story.addToList("category","Stargate: Atlantis")
         else:
             self.story.addToList("category","Stargate: SG-1")
@@ -95,7 +96,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -115,7 +116,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -149,7 +150,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -220,7 +221,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -242,7 +243,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -85,7 +86,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -105,7 +106,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -235,7 +236,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_siyecouk.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -82,7 +83,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         # sites skip that for one-chapter stories.
         # Except it doesn't this time. :-/
         url = self.url #+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -224,7 +225,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         # soup = bs.BeautifulSoup(self._fetchUrl(url))
         # BeautifulSoup objects to <p> inside <span>, which

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -58,7 +59,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -75,7 +76,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.squidge.org'
     
-    @classmethod # must be @staticmethod, don't remove it.
+    @classmethod # must be @classmethod, don't remove it.
     def getConfigSection(cls):
         # The config section name.  Only override if != site domain.
         return cls.getSiteDomain()+'/peja'
@@ -90,7 +91,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -223,7 +224,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -76,7 +77,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -213,7 +214,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
@@ -76,7 +77,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -133,7 +134,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_svufictioncom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -92,13 +93,13 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -120,7 +121,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -145,7 +146,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -258,7 +259,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -43,7 +44,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -81,13 +82,13 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -102,7 +103,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -116,7 +117,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
             # need to log in for this one.
             addurl = "&ageconsent=ok&warning=4"
             url = self.url+'&index=1'+addurl
-            logging.debug("Changing URL: "+url)
+            logger.debug("Changing URL: "+url)
             self.performLogin(url)
             data = self._fetchUrl(url)
 
@@ -229,7 +230,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_test1.py

@@ -18,6 +18,7 @@
 import datetime
 import time
 import logging
+logger = logging.getLogger(__name__)
 
 from .. import BeautifulSoup as bs
 from .. import exceptions
@@ -48,7 +49,7 @@ class TestSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         if self.story.getMetadata('storyId') == '665' and not (self.is_adult or self.getConfig("is_adult")):
-            logging.warn("self.is_adult:%s"%self.is_adult)
+            logger.warn("self.is_adult:%s"%self.is_adult)
             raise exceptions.AdultCheckRequired(self.url)
 
         if self.story.getMetadata('storyId') == '666':
@@ -128,7 +129,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','Comedy')
-        self.story.addToList('genre','SF')
+        self.story.addToList('genre','Sci-Fi')
         self.story.addToList('genre','Noir')
         
         self.story.addToList('characters','Bob Smith')
@@ -184,7 +185,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         if self.story.getMetadata('storyId') == '667':
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
 

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -76,7 +77,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -198,7 +199,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -80,7 +81,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -146,6 +147,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
             self.story.setMetadata('status', 'In-Progress')
 						
         summary = info.find('td', {'class' : 'summary'})
+        summary.name='div' # change td to div so it doesn't mess up the display when using table titlepage.
         self.setDescription(url,summary)
 		
         rating=stripHTML(info.find('td', {'align' : 'left'})).split('(')[1].split(')')[0]
@@ -182,7 +184,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr','img')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_thehookupzonenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -71,7 +72,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -117,13 +118,13 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/CriminalMinds/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -145,7 +146,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -292,7 +293,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_themasquenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,20 +48,20 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         if self.parsedUrl.path.split('/',)[1] == 'wiktt':
             self.story.addToList("category","Harry Potter")
-            self.story.setMetadata('section','/wiktt/efiction/')
+            self.section='/wiktt/efiction/'
             self.dateformat = "%m/%d/%Y"
         else:
             self.story.addToList("category","Originals")
-            self.story.setMetadata('section','/efiction/')
+            self.section='/efiction/'
             self.dateformat = "%b %d, %Y"
             
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + self.story.getMetadata('section') + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + self.section + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','msq')
@@ -98,14 +99,14 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         params['cookiecheck'] = '1'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain()  + self.story.getMetadata('section') + 'user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        loginUrl = 'http://' + self.getSiteDomain()  + self.section + 'user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -127,7 +128,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -152,7 +153,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -186,7 +187,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.story.getMetadata('section') + chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.section + chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -258,7 +259,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_thepetulantpoetesscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
@@ -91,13 +92,13 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "My Account Page" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -110,7 +111,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -223,7 +224,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the  part.  Replace all to remove it usually.
@@ -101,13 +102,13 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -129,7 +130,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -272,7 +273,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -43,7 +44,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -81,13 +82,13 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/library/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -102,7 +103,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -235,7 +236,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time
@@ -40,7 +41,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL("http://"+self.getSiteDomain()\
                          +"/Story-"+self.story.getMetadata('storyId'))
@@ -81,7 +82,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             return
         
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['urealname']))
 
         ## need to pull empty login page first to get ctkn and
@@ -98,7 +99,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         d = self._fetchUrl(loginUrl, params)
     
         if "Stories Published" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -110,7 +111,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         # metadata and chapter list
 
         url=self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # tth won't send you future updates if you aren't 'caught up'
         # on the story.  Login isn't required for F21, but logging in will
@@ -137,7 +138,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 form = soup.find('form', {'id':'sitemaxratingform'})
                 params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
                         'sitemaxrating':'5'}
-                logging.info("Attempting to get rating cookie for %s" % url)
+                logger.info("Attempting to get rating cookie for %s" % url)
                 data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
                 # refetch story page.
                 data = self._fetchUrl(url)
@@ -158,7 +159,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             try:
                 # going to pull part of the meta data from author list page.
                 infourl = 'http://'+self.host+ainfo['href']
-                logging.debug("**StoryInfo** URL: "+infourl)
+                logger.debug("**StoryInfo** URL: "+infourl)
                 infodata = self._fetchUrl(infourl)
                 infosoup = bs.BeautifulSoup(infodata)
 
@@ -175,14 +176,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
     
         try:
             # going to pull part of the meta data from *primary* author list page.
-            logging.debug("**AUTHOR** URL: "+authorurl)
+            logger.debug("**AUTHOR** URL: "+authorurl)
             authordata = self._fetchUrl(authorurl)
             descurl=authorurl
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
                 nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
-                logging.debug("**AUTHOR** nextpage URL: "+nextpage)
+                logger.debug("**AUTHOR** nextpage URL: "+nextpage)
                 authordata = self._fetchUrl(nextpage)
                 descurl=nextpage
                 authorsoup = bs.BeautifulSoup(authordata)
@@ -259,7 +260,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
 
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulSoup(self._fetchUrl(url))
 
         div = soup.find('div', {'id' : 'storyinnerbody'})

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL. http://www.twilightarchives.com/read/9353
         self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
@@ -77,7 +78,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -172,7 +173,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_twilightednet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -42,7 +43,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -83,13 +84,13 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -99,7 +100,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -225,7 +226,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -43,7 +44,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -84,13 +85,13 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -109,7 +110,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -138,12 +139,14 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
+        pagetitlediv = soup.find('div',id='pagetitle')
+        
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        a = pagetitlediv.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitlediv.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
@@ -252,7 +255,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_walkingtheplankorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -45,7 +46,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -84,7 +85,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -215,7 +216,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_whoficcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -56,7 +57,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         # - get chapter list, if not one-shot.
 
         url = self.url+'&chapter=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
         
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
@@ -69,7 +70,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
 
         # pull title(title) and author from the HTML title.
         title = soup.find('title').string
-        logging.debug('Title: %s' % title)
+        logger.debug('Title: %s' % title)
         title = title.split('::')[1].strip()
         self.story.setMetadata('title',title.split(' by ')[0].strip())
         self.story.setMetadata('author',title.split(' by ')[1].strip())
@@ -109,7 +110,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         # <i>Published:</i> 2010.08.15 - <i>Updated:</i> 2010.08.16 - <i>Chapters:</i> 4 - <i>Completed:</i> Yes - <i>Word Count:</i> 4890 </font>
         # </td></tr></table>
              
-        logging.debug("Author URL: "+self.story.getMetadata('authorUrl'))
+        logger.debug("Author URL: "+self.story.getMetadata('authorUrl'))
         soup = bs.BeautifulStoneSoup(self._fetchUrl(self.story.getMetadata('authorUrl')),
                                      selfClosingTags=('br')) # normalize <br> tags to <br />
         # find this story in the list, parse it's metadata based on
@@ -212,7 +213,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -92,13 +93,13 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -120,7 +121,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -145,7 +146,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -286,7 +287,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_wolverineandroguecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/wrfa/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -77,7 +78,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -203,7 +204,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
 
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -86,7 +87,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -209,7 +210,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url))
         

fanficdownloader/adapters/base_adapter.py

@@ -27,6 +27,8 @@ from functools import partial
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 
+logger = logging.getLogger(__name__)
+
 try:
     from google.appengine.api import apiproxy_stub_map
     def urlfetch_timeout_hook(service, call, request, response):
@@ -38,10 +40,10 @@ try:
 
     apiproxy_stub_map.apiproxy.GetPreCallHooks().Append(
         'urlfetch_timeout_hook', urlfetch_timeout_hook, 'urlfetch')
-    logging.info("Hook to make default deadline 10.0 installed.")
+    logger.info("Hook to make default deadline 10.0 installed.")
 except:
     pass
-    #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
+    #logger.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
 
 from ..story import Story
 from ..gziphttp import GZipProcessor
@@ -125,7 +127,7 @@ class BaseSiteAdapter(Configurable):
                 #print code
                 if code == "auto":
                     if not chardet:
-                        logging.info("chardet not available, skipping 'auto' encoding")
+                        logger.info("chardet not available, skipping 'auto' encoding")
                         continue
                     detected = chardet.detect(data)
                     #print detected
@@ -133,12 +135,11 @@ class BaseSiteAdapter(Configurable):
                         code=detected['encoding']
                     else:
                         continue
-                logging.debug("try code:"+code)
                 return data.decode(code)
             except:
-                logging.debug("code failed:"+code)
+                logger.debug("code failed:"+code)
                 pass
-        logging.info("Could not decode story, tried:%s Stripping non-ASCII."%decode)
+        logger.info("Could not decode story, tried:%s Stripping non-ASCII."%decode)
         return "".join([x for x in data if ord(x) < 128])
 
     # Assumes application/x-www-form-urlencoded.  parameters, headers are dict()s
@@ -175,10 +176,10 @@ class BaseSiteAdapter(Configurable):
                 return self._decode(self._fetchUrlRaw(url,parameters))
             except Exception, e:
                 excpt=e
-                logging.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
+                logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
                 
-        logging.error("Giving up on %s" %url)
-        logging.exception(excpt)
+        logger.error("Giving up on %s" %url)
+        logger.exception(excpt)
         raise(excpt)
 
     # Limit chapters to download.  Input starts at 1, list starts at 0
@@ -254,6 +255,13 @@ class BaseSiteAdapter(Configurable):
         "Only needs to be overriden if != site domain."
         return cls.getSiteDomain()
     
+    @classmethod
+    def stripURLParameters(cls,url):
+        "Only needs to be overriden if URL contains more than one parameter"
+        ## remove any trailing '&' parameters--?sid=999 will be left.
+        ## that's all that any of the current adapters need or want.
+        return re.sub(r"&.*$","",url)
+    
     ## URL pattern validation is done *after* picking an adaptor based
     ## on domain instead of *as* the adaptor selector so we can offer
     ## the user example(s) for that particular site.
@@ -304,7 +312,7 @@ class BaseSiteAdapter(Configurable):
         if not fetch:
             fetch=self._fetchUrlRaw
 
-        acceptable_attributes = ['href','name']
+        acceptable_attributes = ['href','name','class','id']
         #print("include_images:"+self.getConfig('include_images'))
         if self.getConfig('include_images'):
             acceptable_attributes.extend(('src','alt','longdesc'))
@@ -356,7 +364,6 @@ class BaseSiteAdapter(Configurable):
 
 def cachedfetch(realfetch,cache,url):
     if url in cache:
-        print("cache hit")
         return cache[url]
     else:
         return realfetch(url)

fanficdownloader/bbcodeutils/bbcodeparser.py

@@ -201,8 +201,11 @@ class bbcodeparser:
     #re_tagArgs = re.compile (r'\s*([^=]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
     re_tagArgs = re.compile (r'\s*([\w]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
 
-    # get a unique name and replace escaped braces 
-    unique = hashlib.md5(code).hexdigest()
+    # get a unique name and replace escaped braces encode utf8 to
+    # prevent CLI/Web from barfing on unicode chars.  Not sure why
+    # this even needs to be 'unique' like this, but that's the way
+    # they wrote it.
+    unique = hashlib.md5(code.encode('utf8')).hexdigest()
     code   = code.replace ('\[', unique+'_OPEN_BRACE')
     code   = code.replace ('\]', unique+'_CLOSE_BRACE')
 

fanficdownloader/configurable.py

@@ -134,7 +134,7 @@ class Configurable(object):
         return self.configuration.hasConfig(key)
         
     def getConfig(self, key, default=""):
-        return self.configuration.getConfig(key)
+        return self.configuration.getConfig(key,default)
 
     def getConfigList(self, key):
         return self.configuration.getConfigList(key)

fanficdownloader/epubutils.py

@@ -22,7 +22,7 @@ def get_dcsource_chaptercount(inputio):
 def get_update_data(inputio,
                     getfilecount=True,
                     getsoups=True):
-    epub = ZipFile(inputio, 'r')
+    epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
 
     ## Find the .opf file.
     container = epub.read("META-INF/container.xml")
@@ -153,7 +153,7 @@ def get_path_part(n):
 def get_story_url_from_html(inputio,_is_good_url=None):
 
     #print("get_story_url_from_html called")
-    epub = ZipFile(inputio, 'r')
+    epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
 
     ## Find the .opf file.
     container = epub.read("META-INF/container.xml")

fanficdownloader/story.py

@@ -20,6 +20,9 @@ import urlparse
 import string
 from math import floor
 from functools import partial
+import logging
+logger = logging.getLogger(__name__)
+import urlparse as up
 
 import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
@@ -27,10 +30,21 @@ from configurable import Configurable
 
 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
+        
+imagetypes = {
+    'jpg':'image/jpeg',
+    'jpeg':'image/jpeg',
+    'png':'image/png',
+    'gif':'image/gif',
+    'svg':'image/svg+xml',
+    }
+
 try:
     from calibre.utils.magick import Image
+    convtype = {'jpg':'JPG', 'png':'PNG'}
 
-    def convert_image(url,data,sizes,grayscale):
+    def convert_image(url,data,sizes,grayscale,
+                      removetrans,imgtype="jpg",background='#ffffff'):
         export = False
         img = Image()
         img.load(data)
@@ -42,18 +56,25 @@ try:
             img.size = (nwidth, nheight)
             export = True
             
+        if normalize_format_name(img.format) != imgtype:
+            export = True
+
+        if removetrans and img.has_transparent_pixels():
+            canvas = Image()
+            canvas.create_canvas(int(img.size[0]), int(img.size[1]), str(background))
+            canvas.compose(img)
+            img = canvas
+            export = True
+            
         if grayscale and img.type != "GrayscaleType":
             img.type = "GrayscaleType"
             export = True
 
-        if normalize_format_name(img.format) != "jpg":
-            export = True
-
         if export:
-            return (img.export('JPG'),'jpg','image/jpeg')
+            return (img.export(convtype[imgtype]),imgtype,imagetypes[imgtype])
         else:
-            print("image used unchanged")
-            return (data,'jpg','image/jpeg')
+            logger.debug("image used unchanged")
+            return (data,imgtype,imagetypes[imgtype])
         
 except:
 
@@ -61,8 +82,9 @@ except:
     try:
         import Image
         from StringIO import StringIO
-        def convert_image(url,data,sizes,grayscale):
-            
+        convtype = {'jpg':'JPEG', 'png':'PNG'}
+        def convert_image(url,data,sizes,grayscale,
+                          removetrans,imgtype="jpg",background='#ffffff'):
             export = False
             img = Image.open(StringIO(data))
             
@@ -73,38 +95,49 @@ except:
                 img = img.resize((nwidth, nheight),Image.ANTIALIAS)
                 export = True
                 
-            if grayscale and img.mode != "L":
-                img = img.convert("L")
-                export = True
-
-            if normalize_format_name(img.format) != "jpg":
+            if normalize_format_name(img.format) != imgtype:
                 if img.mode == "P":
                     # convert pallete gifs to RGB so jpg save doesn't fail.
                     img = img.convert("RGB")
                 export = True
 
+            if removetrans and img.mode == "RGBA":
+                background = Image.new('RGBA', img.size, background)
+                # Paste the image on top of the background
+                background.paste(img, img)
+                img = background.convert('RGB')
+                export = True
+
+            if grayscale and img.mode != "L":
+                img = img.convert("L")
+                export = True
+
             if export:
                 outsio = StringIO()
-                img.save(outsio,'JPEG')
-                return (outsio.getvalue(),'jpg','image/jpeg')
+                img.save(outsio,convtype[imgtype])
+                return (outsio.getvalue(),imgtype,imagetypes[imgtype])
             else:
-                print("image used unchanged")
-                return (data,'jpg','image/jpeg')
+                logger.debug("image used unchanged")
+                return (data,imgtype,imagetypes[imgtype])
         
     except:
-
         # No calibre or PIL, simple pass through with mimetype.
-        imagetypes = {
-            'jpg':'image/jpeg',
-            'jpeg':'image/jpeg',
-            'png':'image/png',
-            'gif':'image/gif',
-            'svg':'image/svg+xml',
-            }
+        def convert_image(url,data,sizes,grayscale,
+                          removetrans,imgtype="jpg",background='#ffffff'):
+            return no_convert_image(url,data)
 
-        def convert_image(url,data,sizes,grayscale):
-            ext=url[url.rfind('.')+1:].lower()
-            return (data,ext,imagetypes[ext])
+## also used for explicit no image processing.
+def no_convert_image(url,data):
+    parsedUrl = up.urlparse(url)
+    
+    ext=parsedUrl.path[parsedUrl.path.rfind('.')+1:].lower()
+    
+    if ext not in imagetypes:
+        logger.debug("no_convert_image url:%s - no known extension"%url)
+        # doesn't have extension? use jpg.
+        ext='jpg'
+        
+    return (data,ext,imagetypes[ext])
         
 def normalize_format_name(fmt):
     if fmt:
@@ -227,7 +260,8 @@ class Story(Configurable):
             self.metadata[key]=value
         if key == "language":
             try:
-                self.metadata['langcode'] = langs[self.metadata[key]]
+                # getMetadata not just self.metadata[] to do replace_metadata.
+                self.metadata['langcode'] = langs[self.getMetadata(key)]
             except:
                 self.metadata['langcode'] = 'en'
         if key == 'dateUpdated':
@@ -240,24 +274,40 @@ class Story(Configurable):
     ## Three part effect only those key(s) lists.
     ## pattern=>replacement
     ## metakey,metakey=>pattern=>replacement
+    ## *Five* part lines.  Effect only when trailing conditional key=>regexp matches
+    ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp]
     def setReplace(self,replace):
         for line in replace.splitlines():
+            (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
+            if "&&" in line:
+                (line,conditional) = map( lambda x: x.strip(), line.split("&&") )
+                (condkey,condregexp) = map( lambda x: x.strip(), conditional.split("=>") )
             if "=>" in line:
                 parts = map( lambda x: x.strip(), line.split("=>") )
                 if len(parts) > 2:
-                    parts[0] = map( lambda x: x.strip(), parts[0].split(",") )
-                    self.replacements.append(parts)
+                    metakeys = map( lambda x: x.strip(), parts[0].split(",") )
+                    (regexp,replacement)=parts[1:]
                 else:
-                    self.replacements.append([None]+parts)
+                    (regexp,replacement)=parts
+
+            if regexp:
+                regexp = re.compile(regexp)
+                if condregexp:
+                    condregexp = re.compile(condregexp)
+                self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
     
     def doReplacments(self,value,key):
-        for (keys,p,v) in self.replacements:
-            if (keys == None or key in keys) \
+        for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
+            if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
-                    and re.search(p,value):
-                #pv=value
-                value = re.sub(p,v,value)
-                #print("change:%s => %s === %s => %s "%(p,v,pv,value))
+                    and regexp.search(value):
+                doreplace=True
+                if condkey and condkey != key: # prevent infinite recursion.
+                    condval = self.getMetadata(condkey)
+                    doreplace = condval != None and condregexp.search(condval)
+                    
+                if doreplace:
+                    value = regexp.sub(replacement,value)
         return value
         
     def getMetadataRaw(self,key):
@@ -280,7 +330,9 @@ class Story(Configurable):
                     value = commaGroups(value)
                 if key == "numChapters":
                     value = commaGroups("%d"%value)
-                if key in ("dateCreated","datePublished","dateUpdated"):
+                if key in ("dateCreated"):
+                    value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d %H:%M:%S"))
+                if key in ("datePublished","dateUpdated"):
                     value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))
 
             if doreplacements:
@@ -411,11 +463,14 @@ class Story(Configurable):
             title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
         self.chapters.append( (title,html) )
 
-    def getChapters(self):
+    def getChapters(self,fortoc=False):
         "Chapters will be tuples of (title,html)"
         retval = []
-        if self.getConfig('add_chapter_numbers') and \
-                self.getConfig('chapter_title_add_pattern'):
+        ## only add numbers if more than one chapter.
+        if len(self.chapters) > 1 and \
+                (self.getConfig('add_chapter_numbers') == "true" \
+                     or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \
+                     and self.getConfig('chapter_title_add_pattern'):
             for index, (title,html) in enumerate(self.chapters):
                 retval.append( (string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),html) ) 
         else:
@@ -434,7 +489,7 @@ class Story(Configurable):
             pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
             for k in origvalues.keys():
                 values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
-
+    
         return string.Template(template).substitute(values).encode('utf8')
 
     # pass fetch in from adapter in case we need the cookies collected
@@ -480,17 +535,31 @@ class Story(Configurable):
         prefix='ffdl'
         if imgurl not in self.imgurls:
             parsedUrl = urlparse.urlparse(imgurl)
+
             try:
-                sizes = [ int(x) for x in self.getConfigList('image_max_size') ]
+                if self.getConfig('no_image_processing'):
+                    (data,ext,mime) = no_convert_image(imgurl,
+                                                       fetch(imgurl))
+                else:
+                    try:
+                        sizes = [ int(x) for x in self.getConfigList('image_max_size') ]
+                    except Exception, e:
+                        raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(self.getConfigList('image_max_size'),e))
+                    grayscale = self.getConfig('grayscale_images')
+                    imgtype = self.getConfig('convert_images_to')
+                    if not imgtype:
+                        imgtype = "jpg"
+                    removetrans = self.getConfig('remove_transparency')
+                    removetrans = removetrans or grayscale or imgtype=="jpg"
+                    (data,ext,mime) = convert_image(imgurl,
+                                                    fetch(imgurl),
+                                                    sizes,
+                                                    grayscale,
+                                                    removetrans,
+                                                    imgtype,
+                                                    background="#"+self.getConfig('background_color'))
             except Exception, e:
-                raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(self.getConfigList('image_max_size'),e))
-            try:
-                (data,ext,mime) = convert_image(imgurl,
-                                                fetch(imgurl),
-                                                sizes,
-                                                self.getConfig('grayscale_images'))
-            except Exception, e:
-                print("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
+                logger.info("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
                 return "failedtoload"
             
             # explicit cover, make the first image.
@@ -525,7 +594,7 @@ class Story(Configurable):
                     ext)
                 self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                 
-            print("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
+            logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
         else:
             newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
             

fanficdownloader/writers/base_writer.py

@@ -18,6 +18,7 @@
 import re
 import os.path
 import datetime
+import string
 import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_DEFLATED
@@ -26,6 +27,8 @@ import logging
 from ..configurable import Configurable
 from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
 
+logger = logging.getLogger(__name__)
+
 class BaseStoryWriter(Configurable):
 
     @staticmethod
@@ -101,6 +104,22 @@ class BaseStoryWriter(Configurable):
         names as Story.metadata, but ENTRY should use label and value.
         """
         if self.getConfig("include_titlepage"):
+
+            if self.hasConfig("titlepage_start"):
+                START = string.Template(self.getConfig("titlepage_start"))
+
+            if self.hasConfig("titlepage_entry"):
+                ENTRY = string.Template(self.getConfig("titlepage_entry"))
+
+            if self.hasConfig("titlepage_end"):
+                END = string.Template(self.getConfig("titlepage_end"))
+
+            if self.hasConfig("titlepage_wide_entry"):
+                WIDE_ENTRY = string.Template(self.getConfig("titlepage_wide_entry"))
+
+            if self.hasConfig("titlepage_no_title_entry"):
+                NO_TITLE_ENTRY = string.Template(self.getConfig("titlepage_no_title_entry"))
+            
             self._write(out,START.substitute(self.story.getAllMetadata()))
 
             if WIDE_ENTRY==None:
@@ -120,11 +139,11 @@ class BaseStoryWriter(Configurable):
                         if self.hasConfig(entry+"_label"):
                             label=self.getConfig(entry+"_label")
                         elif entry in self.titleLabels:
-                            logging.debug("Using fallback label for %s_label"%entry)
+                            logger.debug("Using fallback label for %s_label"%entry)
                             label=self.titleLabels[entry]
                         else:
                             label="%s"%entry.title()
-                            logging.debug("No known label for %s, fallback to '%s'"%(entry,label))
+                            logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                         # If the label for the title entry is empty, use the
                         # 'no title' option if there is one.
@@ -132,6 +151,7 @@ class BaseStoryWriter(Configurable):
                            TEMPLATE= NO_TITLE_ENTRY
                            
                         self._write(out,TEMPLATE.substitute({'label':label,
+                                                             'id':entry,
                                                              'value':self.story.getMetadata(entry)}))
                 else:
                     self._write(out, entry)
@@ -146,11 +166,22 @@ class BaseStoryWriter(Configurable):
         """
         # Only do TOC if there's more than one chapter and it's configured.
         if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
+            if self.hasConfig("tocpage_start"):
+                START = string.Template(self.getConfig("tocpage_start"))
+
+            if self.hasConfig("tocpage_entry"):
+                ENTRY = string.Template(self.getConfig("tocpage_entry"))
+
+            if self.hasConfig("tocpage_end"):
+                END = string.Template(self.getConfig("tocpage_end"))
+            
             self._write(out,START.substitute(self.story.getAllMetadata()))
 
-            for index, (title,html) in enumerate(self.story.getChapters()):
+            for index, (title,html) in enumerate(self.story.getChapters(fortoc=True)):
                 if html:
-                    self._write(out,ENTRY.substitute({'chapter':title, 'index':"%04d"%(index+1)}))
+                    self._write(out,ENTRY.substitute({'chapter':title,
+                                                      'number':index+1,
+                                                      'index':"%04d"%(index+1)}))
 
             self._write(out,END.substitute(self.story.getAllMetadata()))
 
@@ -161,6 +192,8 @@ class BaseStoryWriter(Configurable):
         if outfilename == None:
             outfilename=self.getOutputFileName()
 
+        self.outfilename = outfilename
+
         # minor cheat, tucking css into metadata.
         if self.getConfig("output_css"):
             self.story.setMetadata("output_css",
@@ -171,11 +204,11 @@ class BaseStoryWriter(Configurable):
             
         if not outstream:
             close=True
-            logging.info("Save directly to file: %s" % outfilename)
+            logger.info("Save directly to file: %s" % outfilename)
             if self.getConfig('make_directories'):
                 path=""
-                dirs = os.path.dirname(outfilename).split('/')
-                for dir in dirs:
+                outputdirs = os.path.dirname(outfilename).split('/')
+                for dir in outputdirs:
                     path+=dir+"/"
                     if not os.path.exists(path):
                         os.mkdir(path) ## os.makedirs() doesn't work in 2.5.2?
@@ -198,7 +231,7 @@ class BaseStoryWriter(Configurable):
             outstream = open(outfilename,"wb")
         else:
             close=False
-            logging.debug("Save to stream")
+            logger.debug("Save to stream")
 
         if not metaonly:
             self.story = self.adapter.getStory() # get full story now,
@@ -209,14 +242,14 @@ class BaseStoryWriter(Configurable):
                                                  # fetch once.
         if self.getConfig('zip_output'):
             out = StringIO.StringIO()
+            self.zipout = ZipFile(outstream, 'w', compression=ZIP_DEFLATED)
             self.writeStoryImpl(out)
-            zipout = ZipFile(outstream, 'w', compression=ZIP_DEFLATED)
-            zipout.writestr(self.getBaseFileName(),out.getvalue())
+            self.zipout.writestr(self.getBaseFileName(),out.getvalue())
             # declares all the files created by Windows.  otherwise, when
             # it runs in appengine, windows unzips the files as 000 perms.
-            for zf in zipout.filelist:
+            for zf in self.zipout.filelist:
                 zf.create_system = 0
-            zipout.close()
+            self.zipout.close()
             out.close()
         else:
             self.writeStoryImpl(outstream)
@@ -224,6 +257,27 @@ class BaseStoryWriter(Configurable):
         if close:
             outstream.close()
 
+    def writeFile(self, filename, data):
+        logger.debug("writeFile:%s"%filename)
+        
+        if self.getConfig('zip_output'):
+            outputdirs = os.path.dirname(self.getBaseFileName())
+            if outputdirs:
+                filename=outputdirs+'/'+filename
+            self.zipout.writestr(filename,data)
+        else:
+            outputdirs = os.path.dirname(self.outfilename)
+            if outputdirs:
+                filename=outputdirs+'/'+filename
+
+            dir = os.path.dirname(filename)
+            if not os.path.exists(dir):
+                os.mkdir(dir) ## os.makedirs() doesn't work in 2.5.2?
+                    
+            outstream = open(filename,"wb")
+            outstream.write(data)
+            outstream.close()
+
     def writeStoryImpl(self, out):
         "Must be overriden by sub classes."
         pass

fanficdownloader/writers/writer_epub.py

@@ -29,6 +29,8 @@ from xml.dom.minidom import parse, parseString, getDOMImplementation
 from base_writer import *
 from ..htmlcleanup import stripHTML
 
+logger = logging.getLogger(__name__)
+
 class EpubWriter(BaseStoryWriter):
 
     @staticmethod
@@ -151,8 +153,16 @@ ${value}<br />
 <h3>Update Log</h3>
 ''')
 
+        self.EPUB_LOG_UPDATE_START = string.Template('''
+<p class='log_entry'>
+''')
+                          
         self.EPUB_LOG_ENTRY = string.Template('''
 <b>${label}:</b> <span id="${id}">${value}</span>
+''')
+                          
+        self.EPUB_LOG_UPDATE_END = string.Template('''
+</p><hr />
 ''')
                           
         self.EPUB_LOG_PAGE_END = string.Template('''
@@ -160,30 +170,50 @@ ${value}<br />
 </html>
 ''')
 
+        self.EPUB_LOG_PAGE_END = string.Template('''
+</body>
+</html>
+''')
+
+        self.EPUB_COVER = string.Template('''
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head><title>Cover</title><style type="text/css" title="override_css">
+@page {padding: 0pt; margin:0pt}
+body { text-align: center; padding:0pt; margin: 0pt; }
+div { margin: 0pt; padding: 0pt; }
+</style></head><body><div>
+<img src="${coverimg}" alt="cover"/>
+</div></body></html>
+''')
+        
     def writeLogPage(self, out):
         """
-       XXX 
-
-        
         Write the log page, but only include entries that there's
-        metadata for.  START, ENTRY and END are expected to already by
+        metadata for.  START, ENTRY and END are expected to already be
         string.Template().  START and END are expected to use the same
         names as Story.metadata, but ENTRY should use id, label and value.
         """
-        if self.getConfig("include_logpage"):
+        if self.hasConfig("logpage_start"):
+            START = string.Template(self.getConfig("logpage_start"))
+        else:
+            START = self.EPUB_LOG_PAGE_START
 
-            # if there's a self.story.logfile, there's an existing log
-            # to add to.
-            if self.story.logfile:
-                print("existing logfile found, appending")
-                print("existing data:%s"%self._getLastLogData(self.story.logfile))
-                replace_string = "</body>" # "</h3>"
-                self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string))
-            else:
-                # otherwise, write a new one.
-                self._write(out,self.EPUB_LOG_PAGE_START.substitute(self.story.getAllMetadata()))
-                self._write(out,self._makeLogEntry())
-                self._write(out,self.EPUB_LOG_PAGE_END.substitute(self.story.getAllMetadata()))
+        if self.hasConfig("logpage_end"):
+            END = string.Template(self.getConfig("logpage_end"))
+        else:
+            END = self.EPUB_LOG_PAGE_END
+        
+        # if there's a self.story.logfile, there's an existing log
+        # to add to.
+        if self.story.logfile:
+            logger.debug("existing logfile found, appending")
+            logger.debug("existing data:%s"%self._getLastLogData(self.story.logfile))
+            replace_string = "</body>" # "</h3>"
+            self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string))
+        else:
+            # otherwise, write a new one.
+            self._write(out,START.substitute(self.story.getAllMetadata()))
+            self._write(out,self._makeLogEntry())
+            self._write(out,END.substitute(self.story.getAllMetadata()))
 
     # self parsing instead of Soup because it should be simple and not
     # worth the overhead.
@@ -206,7 +236,22 @@ ${value}<br />
         return values
         
     def _makeLogEntry(self, oldvalues={}):
-        retval = "<p class='log_entry'>"
+        if self.hasConfig("logpage_update_start"):
+            START = string.Template(self.getConfig("logpage_update_start"))
+        else:
+            START = self.EPUB_LOG_UPDATE_START
+
+        if self.hasConfig("logpage_entry"):
+            ENTRY = string.Template(self.getConfig("logpage_entry"))
+        else:
+            ENTRY = self.EPUB_LOG_ENTRY
+
+        if self.hasConfig("logpage_update_end"):
+            END = string.Template(self.getConfig("logpage_update_end"))
+        else:
+            END = self.EPUB_LOG_UPDATE_END
+
+        retval = START.substitute(self.story.getAllMetadata())
 
         for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"):
             if self.isValidMetaEntry(entry):
@@ -215,22 +260,22 @@ ${value}<br />
                     if self.hasConfig(entry+"_label"):
                         label=self.getConfig(entry+"_label")
                     elif entry in self.titleLabels:
-                        logging.debug("Using fallback label for %s_label"%entry)
+                        logger.debug("Using fallback label for %s_label"%entry)
                         label=self.titleLabels[entry]
                     else:
                         label="%s"%entry.title()
-                        logging.debug("No known label for %s, fallback to '%s'"%(entry,label))
+                        logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
-                    retval = retval + self.EPUB_LOG_ENTRY.substitute({'id':entry,
-                                                                      'label':label,
-                                                                      'value':val})
+                    retval = retval + ENTRY.substitute({'id':entry,
+                                                        'label':label,
+                                                        'value':val})
             else:
                 # could be useful for introducing extra text, but
                 # mostly it makes it easy to tell when you get the
                 # keyword wrong.
                 retval = retval + entry
                 
-        retval = retval + "</p><hr />"
+        retval = retval + END.substitute(self.story.getAllMetadata())
         
         if self.getConfig('replace_hr'):
             retval = retval.replace("<hr />","<div class='center'>* * *</div>")
@@ -368,9 +413,9 @@ ${value}<br />
         guide = None
         coverIO = None
                 
-        imgid = "image0000"
+        coverimgid = "image0000"
         if not self.story.cover and self.story.oldcover:
-            print("writer_epub: no new cover, has old cover, write image.")
+            logger.debug("writer_epub: no new cover, has old cover, write image.")
             (oldcoverhtmlhref,
              oldcoverhtmltype,
              oldcoverhtmldata,
@@ -380,8 +425,8 @@ ${value}<br />
             outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata)
             outputepub.writestr(oldcoverimghref,oldcoverimgdata)
             
-            imgid = "image0"
-            items.append((imgid,
+            coverimgid = "image0"
+            items.append((coverimgid,
                           oldcoverimghref,
                           oldcoverimgtype,
                           None))
@@ -406,6 +451,10 @@ ${value}<br />
                               imgmap['mime'],
                               None))
                 imgcount+=1
+                if 'cover' in imgfile:
+                    # make sure coverimgid is set to the cover, not
+                    # just the first image.
+                    coverimgid = items[-1][0]
 
         
         items.append(("style","OEBPS/stylesheet.css","text/css",None))
@@ -417,7 +466,7 @@ ${value}<br />
             itemrefs.append("cover")
             # 
             # <meta name="cover" content="cover.jpg"/>
-            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0000",
+            metadata.appendChild(newTag(contentdom,"meta",{"content":coverimgid,
                                                            "name":"cover"}))
             # cover stuff for later:
             # at end of <package>:
@@ -429,16 +478,12 @@ ${value}<br />
                                                        "title":"Cover",
                                                        "href":"OEBPS/cover.xhtml"}))
             
+            if self.hasConfig("cover_content"):
+                COVER = string.Template(self.getConfig("cover_content"))
+            else:
+                COVER = self.EPUB_COVER
             coverIO = StringIO.StringIO()
-            coverIO.write('''
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head><title>Cover</title><style type="text/css" title="override_css">
-@page {padding: 0pt; margin:0pt}
-body { text-align: center; padding:0pt; margin: 0pt; }
-div { margin: 0pt; padding: 0pt; }
-</style></head><body><div>
-<img src="%s" alt="cover"/>
-</div></body></html>
-'''%self.story.cover)
+            coverIO.write(COVER.substitute(dict(self.story.getAllMetadata().items()+{'coverimg':self.story.cover}.items())))
             
         if self.getConfig("include_titlepage"):
             items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page"))
@@ -447,11 +492,15 @@ div { margin: 0pt; padding: 0pt; }
             items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
             itemrefs.append("toc_page")
 
-        if self.getConfig("include_logpage"):
+        dologpage = ( self.getConfig("include_logpage") == "smart" and \
+                          (self.story.logfile or self.story.getMetadataRaw("status") == "In-Progress") )  \
+                     or self.getConfig("include_logpage") == "true"
+
+        if dologpage:
             items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
             itemrefs.append("log_page")
             
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (title,html) in enumerate(self.story.getChapters(fortoc=True)):
             if html:
                 i=index+1
                 items.append(("file%04d"%i,
@@ -483,8 +532,8 @@ div { margin: 0pt; padding: 0pt; }
         contentxml = contentdom.toxml(encoding='utf-8')
         
         # tweak for brain damaged Nook STR.  Nook insists on name before content.
-        contentxml = contentxml.replace('<meta content="%s" name="cover"/>'%imgid,
-                                        '<meta name="cover" content="%s"/>'%imgid)
+        contentxml = contentxml.replace('<meta content="%s" name="cover"/>'%coverimgid,
+                                        '<meta name="cover" content="%s"/>'%coverimgid)
         outputepub.writestr("content.opf",contentxml)
 
         contentdom.unlink()
@@ -582,17 +631,28 @@ div { margin: 0pt; padding: 0pt; }
             outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
         tocpageIO.close()
 
-        # write log page.
-        logpageIO = StringIO.StringIO()
-        self.writeLogPage(logpageIO)
-        if logpageIO.getvalue(): # will be false if no log page.
+        if dologpage:
+            # write log page.
+            logpageIO = StringIO.StringIO()
+            self.writeLogPage(logpageIO)
             outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
-        logpageIO.close()
+            logpageIO.close()
+
+        if self.hasConfig('chapter_start'):
+            CHAPTER_START = string.Template(self.getConfig("chapter_start"))
+        else:
+            CHAPTER_START = self.EPUB_CHAPTER_START
+        
+        if self.hasConfig('chapter_end'):
+            CHAPTER_END = string.Template(self.getConfig("chapter_end"))
+        else:
+            CHAPTER_END = self.EPUB_CHAPTER_END
         
         for index, (title,html) in enumerate(self.story.getChapters()):
             if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                fullhtml = self.EPUB_CHAPTER_START.substitute({'chapter':title, 'index':index+1}) + html + self.EPUB_CHAPTER_END.substitute({'chapter':title, 'index':index+1})
+                logger.debug('Writing chapter text for: %s' % title)
+                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big

fanficdownloader/writers/writer_html.py

@@ -46,6 +46,10 @@ ${output_css}
 <h1><a href="${storyUrl}">${title}</a> by ${authorHTML}</h1>
 ''')
 
+        self.HTML_COVER = string.Template('''
+<img src="${coverimg}" alt="cover" />
+''')
+        
         self.HTML_TITLE_PAGE_START = string.Template('''
 <table class="full">
 ''')
@@ -75,6 +79,8 @@ ${output_css}
 <a name="section${index}"><h2>${chapter}</h2></a>
 ''')
 
+        self.HTML_CHAPTER_END = string.Template('')
+
         self.HTML_FILE_END = string.Template('''
 </body>
 </html>''')
@@ -82,8 +88,26 @@ ${output_css}
 
     def writeStoryImpl(self, out):
 
-        self._write(out,self.HTML_FILE_START.substitute(self.story.getAllMetadata()))
+        if self.hasConfig("cover_content"):
+            COVER = string.Template(self.getConfig("cover_content"))
+        else:
+            COVER = self.HTML_COVER
 
+        if self.hasConfig('file_start'):
+            FILE_START = string.Template(self.getConfig("file_start"))
+        else:
+            FILE_START = self.HTML_FILE_START
+
+        if self.hasConfig('file_end'):
+            FILE_END = string.Template(self.getConfig("file_end"))
+        else:
+            FILE_END = self.HTML_FILE_END
+        
+        self._write(out,FILE_START.substitute(self.story.getAllMetadata()))
+
+        if self.getConfig('include_images') and self.story.cover:
+            self._write(out,COVER.substitute(dict(self.story.getAllMetadata().items()+{'coverimg':self.story.cover}.items())))
+            
         self.writeTitlePage(out,
                             self.HTML_TITLE_PAGE_START,
                             self.HTML_TITLE_ENTRY,
@@ -94,10 +118,27 @@ ${output_css}
                           self.HTML_TOC_ENTRY,
                           self.HTML_TOC_PAGE_END)
 
+        if self.hasConfig('chapter_start'):
+            CHAPTER_START = string.Template(self.getConfig("chapter_start"))
+        else:
+            CHAPTER_START = self.HTML_CHAPTER_START
+        
+        if self.hasConfig('chapter_end'):
+            CHAPTER_END = string.Template(self.getConfig("chapter_end"))
+        else:
+            CHAPTER_END = self.HTML_CHAPTER_END
+        
         for index, (title,html) in enumerate(self.story.getChapters()):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
-                self._write(out,self.HTML_CHAPTER_START.substitute({'chapter':title, 'index':"%04d"%(index+1)}))
+                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                self._write(out,CHAPTER_START.substitute(vals))
                 self._write(out,html)
+                self._write(out,CHAPTER_END.substitute(vals))
 
-        self._write(out,self.HTML_FILE_END.substitute(self.story.getAllMetadata()))
+        self._write(out,FILE_END.substitute(self.story.getAllMetadata()))
+
+        if self.getConfig('include_images'):
+            for imgmap in self.story.getImgUrls():
+                self.writeFile(imgmap['newsrc'],imgmap['data'])
+        

fanficdownloader/writers/writer_mobi.py

@@ -88,27 +88,6 @@ ${value}<br />
         self.MOBI_TABLE_TITLE_PAGE_END = string.Template('''
 </table>
 
-</body>
-</html>
-''')
-
-        self.MOBI_TOC_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
-<title>${title} by ${author}</title>
-</head>
-<body>
-<div>
-<h3>Table of Contents</h3>
-''')
-
-        self.MOBI_TOC_ENTRY = string.Template('''
-<a href="file${index}.xhtml">${chapter}</a><br />
-''')
-                          
-        self.MOBI_TOC_PAGE_END = string.Template('''
-</div>
 </body>
 </html>
 ''')
@@ -169,10 +148,21 @@ ${value}<br />
         #     files.append(tocpageIO.getvalue())
         # tocpageIO.close()
 
+        if self.hasConfig('chapter_start'):
+            CHAPTER_START = string.Template(self.getConfig("chapter_start"))
+        else:
+            CHAPTER_START = self.MOBI_CHAPTER_START
+        
+        if self.hasConfig('chapter_end'):
+            CHAPTER_END = string.Template(self.getConfig("chapter_end"))
+        else:
+            CHAPTER_END = self.MOBI_CHAPTER_END
+        
         for index, (title,html) in enumerate(self.story.getChapters()):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
-                fullhtml = self.MOBI_CHAPTER_START.substitute({'chapter':title, 'index':index+1}) + html + self.MOBI_CHAPTER_END.substitute({'chapter':title, 'index':index+1})
+                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big