Merge fimfiction changes to trunk.

Add DB error to mw.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 2-0-10
+version: 4-4-7
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -1,28 +1,13 @@
 #!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 # -*- coding: utf-8 -*-
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2013, Jim Miller'
+__copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import sys
-if sys.version_info >= (2, 7):
-    import logging
-    logger = logging.getLogger(__name__)
-    loghandler=logging.StreamHandler()
-    loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-    logger.addHandler(loghandler)
-    loghandler.setLevel(logging.DEBUG)
-    logger.setLevel(logging.DEBUG)
-
-# pulls in translation files for _() strings
-try:
-    load_translations()
-except NameError:
-    pass # load_translations() added in calibre 1.9
-
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
@@ -39,11 +24,11 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     calibre utilities to run without needing to load the GUI libraries.
     '''
     name                = 'FanFictionDownLoader'
-    description         = _('UI plugin to download FanFiction stories from various sites.')
+    description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 0, 10)
-    minimum_calibre_version = (1, 48, 0)
+    version             = (1, 5, 15)
+    minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name
@@ -95,29 +80,11 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
         if ac is not None:
             ac.apply_settings()
 
-    def cli_main(self,argv):
-        # I believe there's no performance hit loading these here when
-        # CLI--it would load everytime anyway.
-        from StringIO import StringIO
-        from calibre.library import db
-        from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
-        from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
-        from calibre.utils.config import prefs as calibre_prefs
-        from optparse import OptionParser      
-
-        parser = OptionParser('%prog --run-plugin '+self.name+' -- [options] <storyurl>')
-        parser.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
-        # parser.add_option('--dont-notify-gui', default=False, action='store_true',
-        #               help=_('Do not notify the running calibre GUI (if any) that the database has'
-        #                      ' changed. Use with care, as it can lead to database corruption!'))
-
-        pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
-                 or not x.startswith('-')]
-        opts, args = parser.parse_args(pargs)
-
-        ffdl_prefs = PrefsFacade(db(path=opts.library_path,
-                                    read_only=True))
-        ffdl_main(argv[1:],
-                  parser=parser,
-                  passed_defaultsini=StringIO(get_resources("defaults.ini")),
-                  passed_personalini=StringIO(ffdl_prefs["personal.ini"]))
+# For testing, run from command line with this:
+# calibre-debug -e __init__.py
+# 
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    from calibre.gui2.preferences import test_widget
+    app = QApplication([])
+    test_widget('Advanced', 'Plugins')

calibre-plugin/about.txt

@@ -1,6 +1,6 @@
 <hr />
 
-<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
+<p>Created by Jim Miller, borrowing heavily from Grant Drake's
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
 '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'

calibre-plugin/common_utils.py

@@ -8,21 +8,12 @@ __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
-try:
-    from PyQt5 import QtWidgets as QtGui
-    from PyQt5.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                          QTextEdit, QListWidget, QAbstractItemView)
-except ImportError as e:
-    from PyQt4 import QtGui
-    from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                          QTextEdit, QListWidget, QAbstractItemView)
-    
+from PyQt4 import QtGui
+from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime)
 from calibre.constants import iswindows
-from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
+from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME
 from calibre.gui2.actions import menu_action_unique_name
 from calibre.gui2.keyboard import ShortcutConfig
 from calibre.utils.config import config_dir
@@ -203,7 +194,7 @@ class ImageTitleLayout(QHBoxLayout):
     '''
     A reusable layout widget displaying an image followed by a title
     '''
-    def __init__(self, parent, icon_name, title, tooltip=None):
+    def __init__(self, parent, icon_name, title):
         QHBoxLayout.__init__(self)
         title_image_label = QLabel(parent)
         pixmap = get_pixmap(icon_name)
@@ -224,9 +215,6 @@ class ImageTitleLayout(QHBoxLayout):
         self.addWidget(shelf_label)
         self.insertStretch(-1)
 
-        if tooltip:
-            title_image_label.setToolTip(tooltip)
-            shelf_label.setToolTip(tooltip)
 
 class SizePersistedDialog(QDialog):
     '''
@@ -246,8 +234,8 @@ class SizePersistedDialog(QDialog):
             self.restoreGeometry(self.geom)
 
     def dialog_closing(self, result):
-        self.geom = bytearray(self.saveGeometry())
-        gprefs[self.unique_pref_name] = self.geom
+        geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = geom
 
 
 class ReadOnlyTableWidgetItem(QTableWidgetItem):
@@ -457,97 +445,3 @@ class DateDelegate(QStyledItemDelegate):
             model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
         else:
             model.setData(index, QDateTime(val), Qt.EditRole)
-            
-class PrefsViewerDialog(SizePersistedDialog):
-
-    def __init__(self, gui, namespace):
-        SizePersistedDialog.__init__(self, gui, 'Prefs Viewer dialog')
-        self.setWindowTitle('Preferences for: '+namespace)
-
-        self.gui = gui
-        self.db = gui.current_db
-        self.namespace = namespace
-        self._init_controls()
-        self.resize_dialog()
-
-        self._populate_settings()
-
-        if self.keys_list.count():
-            self.keys_list.setCurrentRow(0)
-
-    def _init_controls(self):
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-
-        ml = QHBoxLayout()
-        layout.addLayout(ml, 1)
-
-        self.keys_list = QListWidget(self)
-        self.keys_list.setSelectionMode(QAbstractItemView.SingleSelection)
-        self.keys_list.setFixedWidth(150)
-        self.keys_list.setAlternatingRowColors(True)
-        ml.addWidget(self.keys_list)
-        self.value_text = QTextEdit(self)
-        self.value_text.setTabStopWidth(24)
-        self.value_text.setReadOnly(True)
-        ml.addWidget(self.value_text, 1)
-
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok)
-        button_box.accepted.connect(self.accept)
-        self.clear_button = button_box.addButton('Clear', QDialogButtonBox.ResetRole)
-        self.clear_button.setIcon(get_icon('trash.png'))
-        self.clear_button.setToolTip('Clear all settings for this plugin')
-        self.clear_button.clicked.connect(self._clear_settings)
-        layout.addWidget(button_box)
-
-    def _populate_settings(self):
-        self.keys_list.clear()
-        ns_prefix = self._get_ns_prefix()
-        keys = sorted([k[len(ns_prefix):] for k in self.db.prefs.iterkeys()
-                       if k.startswith(ns_prefix)])
-        for key in keys:
-            self.keys_list.addItem(key)
-        self.keys_list.setMinimumWidth(self.keys_list.sizeHintForColumn(0))
-        self.keys_list.currentRowChanged[int].connect(self._current_row_changed)
-
-    def _current_row_changed(self, new_row):
-        if new_row < 0:
-            self.value_text.clear()
-            return
-        key = unicode(self.keys_list.currentItem().text())
-        val = self.db.prefs.get_namespaced(self.namespace, key, '')
-        self.value_text.setPlainText(self.db.prefs.to_raw(val))
-        
-    def _get_ns_prefix(self):
-        return 'namespaced:%s:'% self.namespace
-        
-    def _clear_settings(self):
-        from calibre.gui2.dialogs.confirm_delete import confirm
-        message = '<p>Are you sure you want to clear your settings in this library for this plugin?</p>' \
-                  '<p>Any settings in other libraries or stored in a JSON file in your calibre plugins ' \
-                  'folder will not be touched.</p>' \
-                  '<p>You must restart calibre afterwards.</p>'
-        if not confirm(message, self.namespace+'_clear_settings', self):
-            return
-        ns_prefix = self._get_ns_prefix()
-        keys = [k for k in self.db.prefs.iterkeys() if k.startswith(ns_prefix)]
-        for k in keys:
-            del self.db.prefs[k]
-        self._populate_settings()
-        d = info_dialog(self, 'Settings deleted', 
-                        '<p>All settings for this plugin in this library have been cleared.</p>'
-                        '<p>Please restart calibre now.</p>',
-                        show_copy_button=False)
-        b = d.bb.addButton(_('Restart calibre now'), d.bb.AcceptRole)
-        b.setIcon(QIcon(I('lt.png')))
-        d.do_restart = False
-        def rf():
-            d.do_restart = True
-        b.clicked.connect(rf)
-        d.set_details('')
-        d.exec_()
-        b.clicked.disconnect()
-        self.close()
-        if d.do_restart:
-            self.gui.quit(restart=True)
-        

calibre-plugin/ffdl_util.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2013, Jim Miller'
-__docformat__ = 'restructuredtext en'
-
-from StringIO import StringIO
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
-from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)
-
-def get_ffdl_personalini():
-    if prefs['includeimages']:
-        # this is a cheat to make it easier for users.
-        return '''[epub]
-include_images:true
-keep_summary_html:true
-make_firstimage_cover:true
-''' + prefs['personal.ini']
-    else:
-        return prefs['personal.ini']
-
-def get_ffdl_config(url,fileform="epub",personalini=None):
-    if not personalini:
-        personalini = get_ffdl_personalini()
-    site='unknown'
-    try:
-        site = adapters.getConfigSectionFor(url)
-    except Exception as e:
-        print("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
-    configuration = Configuration(site,fileform)
-    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
-    configuration.readfp(StringIO(personalini))
-
-    return configuration
-
-def get_ffdl_adapter(url,fileform="epub",personalini=None):
-    return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
-    

calibre-plugin/jobs.py

@@ -4,32 +4,24 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2014, Jim Miller'
+__copyright__ = '2012, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
-import logging
-logger = logging.getLogger(__name__)
-
 import time, os, traceback
 
+from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
-from calibre.constants import numeric_version as calibre_version
-
-# for smarten punc
-from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
 from calibre.utils.logging import Log
-from collections import namedtuple
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
 
-from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config)
 # ------------------------------------------------------------------------------
 #
 #              Functions to perform downloads using worker jobs
@@ -46,31 +38,31 @@ def do_download_worker(book_list, options,
     '''
     server = Server(pool_size=cpus)
 
-    logger.info(options['version'])
+    print(options['version'])
     total = 0
-    alreadybad = []
     # Queue all the jobs
-    logger.info("Adding jobs for URLs:")
+    print("Adding jobs for URLs:")
     for book in book_list:
-        logger.info("%s"%book['url'])
         if book['good']:
+            print("%s"%book['url'])
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',
                     (book,options)]
-            job = ParallelJob('arbitrary_n',
+            job = ParallelJob('arbitrary',
                               "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
                               done=None,
                               args=args)
             job._book = book
+            # job._book_id = book_id
+            # job._title = title
+            # job._modified_date = modified_date
+            # job._existing_isbn = existing_isbn
             server.add_job(job)
-        else:
-            # was already bad before the subprocess ever started.
-            alreadybad.append(book)
     
     # This server is an arbitrary_n job, so there is a notifier available.
     # Set the % complete to a small number to avoid the 'unavailable' indicator
-    notification(0.01, _('Downloading FanFiction Stories'))
+    notification(0.01, 'Downloading FanFiction Stories')
 
     # dequeue the job results as they arrive, saving the results
     count = 0
@@ -82,20 +74,20 @@ def do_download_worker(book_list, options,
         if not job.is_finished:
             continue
         # A job really finished. Get the information.
+        output_book = job.result
+        #print("output_book:%s"%output_book)
         book_list.remove(job._book)
         book_list.append(job.result)
         book_id = job._book['calibre_id']
+        #title = job._title
         count = count + 1
-        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
+        notification(float(count)/total, 'Downloaded Story')
         # Add this job's output to the current log
-        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        logger.info(job.details)
+        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        print(job.details)
 
         if count >= total:
-            logger.info("\n"+_("Successful:")+"\n%s\n"%("\n".join([book['url'] for book in 
-                                                      filter(lambda x: x['good'], book_list) ] ) ) )
-            logger.info("\n"+_("Unsuccessful:")+"\n%s\n"%("\n".join([book['url'] for book in 
-                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
+            # All done!
             break
 
     server.close()
@@ -103,126 +95,58 @@ def do_download_worker(book_list, options,
     # return the book list as the job result
     return book_list
 
-def do_download_for_worker(book,options,notification=lambda x,y:x):
+def do_download_for_worker(book,options):
     '''
-    Child job, to download story when run as a worker job
+    Child job, to extract isbn from formats for this specific book,
+    when run as a worker job
     '''
     try:
-        book['comment'] = _('Download started...')
+        book['comment'] = 'Download started...'
 
-        configuration = get_ffdl_config(book['url'],
-                                        options['fileform'],
-                                        options['personal.ini'])
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(options['personal.ini']))
         
-        if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-            configuration.set("overrides","never_make_cover","true")
-
-        # images only for epub, html, even if the user mistakenly
-        # turned it on else where.
-        if options['fileform'] not in ("epub","html"):
-            configuration.set("overrides","include_images","false")
-        
-        adapter = adapters.getAdapter(configuration,book['url'])
+        adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
         adapter.is_adult = book['is_adult'] 
         adapter.username = book['username'] 
         adapter.password = book['password']
-        adapter.setChaptersRange(book['begin'],book['end'])
-        
-        adapter.load_cookiejar(options['cookiejarfile'])
-        logger.debug("cookiejar:%s"%adapter.cookiejar)
-        adapter.set_pagecache(options['pagecache'])
         
         story = adapter.getStoryMetadataOnly()
-        if 'calibre_series' in book:
-            adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
-            
-        # set PI version instead of default.
-        if 'version' in options:
-            story.setMetadata('version',options['version'])
-            
-        writer = writers.getWriter(options['fileform'],configuration,adapter)
+        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
 
         outfile = book['outfile']
 
         ## No need to download at all.  Shouldn't ever get down here.
         if options['collision'] in (CALIBREONLY):
-            logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
             book['comment'] = 'Metadata collected.'
             
         ## checks were done earlier, it's new or not dup or newer--just write it.
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
-
-            # preserve logfile even on overwrite.
-            if 'epub_for_update' in book:
-                (urlignore,
-                 chaptercountignore,
-                 oldchaptersignore,
-                 oldimgsignore,
-                 oldcoverignore,
-                 calibrebookmarkignore,
-                 # only logfile set in adapter, so others aren't used.
-                 adapter.logfile) = get_update_data(book['epub_for_update'])
-
-                # change the existing entries id to notid so
-                # write_epub writes a whole new set to indicate overwrite.
-                if adapter.logfile:
-                    adapter.logfile = adapter.logfile.replace("span id","span notid")
-            
-            logger.info("write to %s"%outfile)
+            print("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
             
         ## checks were done earlier, just update it.
         elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-
+            
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
-            urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
-            (url,
-             chaptercount,
+            urlchaptercount = int(story.getMetadata('numChapters'))
+            (url,chaptercount,
              adapter.oldchapters,
-             adapter.oldimgs,
-             adapter.oldcover,
-             adapter.calibrebookmark,
-             adapter.logfile) = get_update_data(book['epub_for_update'])
+             adapter.oldimgs) = get_update_data(book['epub_for_update'])
 
-            # dup handling from ffdl_plugin needed for anthology updates.
-            if options['collision'] == UPDATE:
-                if chaptercount == urlchaptercount:
-                    book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
-                    book['outfile'] = book['epub_for_update'] # for anthology merge ops.
-                    return book
-
-            # dup handling from ffdl_plugin needed for anthology updates.
-            if chaptercount > urlchaptercount:
-                raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
-
-            if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
-                    and adapter.getConfig("do_update_hook"):
-                chaptercount = adapter.hookForUpdates(chaptercount)
-
-            logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            logger.info("write to %s"%outfile)
+            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            print("write to %s"%outfile)
 
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             
-            book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
+            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
                 (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
         
-        if options['smarten_punctuation'] and options['fileform'] == "epub" \
-                and calibre_version >= (0, 9, 39):
-            # do smarten_punctuation from calibre's polish feature
-            data = {'smarten_punctuation':True}
-            opts = ALL_OPTS.copy()
-            opts.update(data)
-            O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
-            opts = O(**opts)
-            
-            log = Log(level=Log.DEBUG)
-            # report = []
-            polish({outfile:outfile}, opts, log, logger.info) # report.append
-        
     except NotGoingToDownload as d:
         book['good']=False
         book['comment']=unicode(d)
@@ -232,9 +156,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
         book['good']=False
         book['comment']=unicode(e)
         book['icon']='dialog_error.png'
-        book['status'] = 'Error'
-        logger.info("Exception: %s:%s"%(book,unicode(e)))
+        print("Exception: %s:%s"%(book,unicode(e)))
         traceback.print_exc()
-        
+
     #time.sleep(10)
     return book

calibre-plugin/prefs.py

@@ -1,150 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2013, Jim Miller'
-__docformat__ = 'restructuredtext en'
-
-import copy
-
-from calibre.utils.config import JSONConfig
-from calibre.gui2.ui import get_gui
-
-from calibre_plugins.fanfictiondownloader_plugin.dialogs import OVERWRITE
-from calibre_plugins.fanfictiondownloader_plugin.common_utils import get_library_uuid
-PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
-PREFS_KEY_SETTINGS = 'settings'
-
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-default_prefs = {}
-default_prefs['personal.ini'] = get_resources('plugin-example.ini')
-default_prefs['rejecturls'] = ''
-default_prefs['rejectreasons'] = '''Sucked
-Boring
-Dup from another site'''
-default_prefs['reject_always'] = False
-
-default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
-default_prefs['updateepubcover'] = False
-default_prefs['keeptags'] = False
-default_prefs['suppressauthorsort'] = False
-default_prefs['suppresstitlesort'] = False
-default_prefs['mark'] = False
-default_prefs['showmarked'] = False
-default_prefs['autoconvert'] = False
-default_prefs['urlsfromclip'] = True
-default_prefs['updatedefault'] = True
-default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = OVERWRITE
-default_prefs['deleteotherforms'] = False
-default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
-default_prefs['lookforurlinhtml'] = False
-default_prefs['checkforseriesurlid'] = True
-default_prefs['checkforurlchange'] = True
-default_prefs['injectseries'] = False
-default_prefs['smarten_punctuation'] = False
-
-default_prefs['send_lists'] = ''
-default_prefs['read_lists'] = ''
-default_prefs['addtolists'] = False
-default_prefs['addtoreadlists'] = False
-default_prefs['addtolistsonread'] = False
-
-default_prefs['gcnewonly'] = False
-default_prefs['gc_site_settings'] = {}
-default_prefs['allow_gc_from_ini'] = True
-default_prefs['gc_polish_cover'] = False
-
-default_prefs['countpagesstats'] = []
-
-default_prefs['errorcol'] = ''
-default_prefs['custom_cols'] = {}
-default_prefs['custom_cols_newonly'] = {}
-default_prefs['allow_custcol_from_ini'] = True
-
-default_prefs['std_cols_newonly'] = {}
-
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-def set_library_config(library_config,db):
-    db.prefs.set_namespaced(PREFS_NAMESPACE,
-                            PREFS_KEY_SETTINGS,
-                            library_config)
-
-def get_library_config(db):
-    library_id = get_library_uuid(db)
-    library_config = None
-    # Check whether this is a configuration needing to be migrated
-    # from json into database.  If so: get it, set it, rename it in json.
-    if library_id in old_prefs:
-        #print("get prefs from old_prefs")
-        library_config = old_prefs[library_id]
-        set_library_config(library_config,db)
-        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
-        del old_prefs[library_id]
-
-    if library_config is None:
-        #print("get prefs from db")
-        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
-                                                 copy.deepcopy(default_prefs))
-    return library_config
-
-# fake out so I don't have to change the prefs calls anywhere.  The
-# Java programmer in me is offended by op-overloading, but it's very
-# tidy.
-class PrefsFacade():
-    def _get_db(self):
-        if self.passed_db:
-            return self.passed_db
-        else:
-            # In the GUI plugin we want current db so we detect when
-            # it's changed.  CLI plugin calls need to pass db in.
-            return get_gui().current_db
-    
-    def __init__(self,passed_db=None):
-        self.default_prefs = default_prefs
-        self.libraryid = None
-        self.current_prefs = None
-        self.passed_db=passed_db
-        
-    def _get_prefs(self):
-        libraryid = get_library_uuid(self._get_db())
-        if self.current_prefs == None or self.libraryid != libraryid:
-            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
-            self.libraryid = libraryid
-            self.current_prefs = get_library_config(self._get_db())
-        return self.current_prefs
-        
-    def __getitem__(self,k):            
-        prefs = self._get_prefs()
-        if k not in prefs:
-            # pulls from default_prefs.defaults automatically if not set
-            # in default_prefs
-            return self.default_prefs[k]
-        return prefs[k]
-
-    def __setitem__(self,k,v):
-        prefs = self._get_prefs()
-        prefs[k]=v
-        # self._save_prefs(prefs)
-
-    def __delitem__(self,k):
-        prefs = self._get_prefs()
-        if k in prefs:
-            del prefs[k]
-
-    def save_to_db(self):
-        set_library_config(self._get_prefs(),self._get_db())
-        
-prefs = PrefsFacade()
-

downloader.py

@@ -15,6 +15,10 @@
 # limitations under the License.
 #
 
+import logging
+## XXX cli option for logging level.
+logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+
 import sys, os
 from os.path import normpath, expanduser, isfile, join
 from StringIO import StringIO
@@ -23,35 +27,9 @@ import getpass
 import string
 import ConfigParser
 from subprocess import call
-import pprint
 
-import logging
-if sys.version_info >= (2, 7):
-    # suppresses default logger.  Logging is setup in fanficdownload/__init__.py so it works in calibre, too.
-    rootlogger = logging.getLogger()
-    loghandler=logging.NullHandler()
-    loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
-    rootlogger.addHandler(loghandler)
-
-try:
-    from calibre.constants import numeric_version as calibre_version
-    is_calibre = True
-except:
-    is_calibre = False
-
-# using try/except directly was masking errors during development.
-if is_calibre:
-    # running under calibre
-    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
-    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
-    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
-else:
-    from fanficdownloader import adapters,writers,exceptions
-    from fanficdownloader.configurable import Configuration
-    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-    from fanficdownloader.geturls import get_urls_from_page
-    
+from fanficdownloader import adapters,writers,exceptions
+from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
@@ -64,23 +42,15 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     del writer
     return output_filename
 
-def main(argv,
-         parser=None,
-         passed_defaultsini=None,
-         passed_personalini=None):
+def main():
    # read in args, anything starting with -- will be treated as --<varible>=<value>
-   if not parser:
-       parser = OptionParser("usage: %prog [options] storyurl")
+   usage = "usage: %prog [options] storyurl"
+   parser = OptionParser(usage)
    parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT")
-
-   if passed_defaultsini:
-       config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini"
-   else:
-       config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini"
+                     help="write story as FORMAT, epub(default), text or html", metavar="FORMAT")
    parser.add_option("-c", "--config",
                      action="append", dest="configfile", default=None,
-                     help=config_help, metavar="CONFIG")
+                     help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG")
    parser.add_option("-b", "--begin", dest="begin", default=None,
                      help="Begin with Chapter START", metavar="START")
    parser.add_option("-e", "--end", dest="end", default=None,
@@ -93,86 +63,29 @@ def main(argv,
                      help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
-                     help="Update an existing epub with new chapters, give epub filename instead of storyurl.",)
-   parser.add_option("--update-cover",
-                     action="store_true", dest="updatecover",
-                     help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
+                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
-                     help="Force overwrite of an existing epub, download and overwrite all chapters.",)
-   parser.add_option("-l", "--list",
-                     action="store_true", dest="list",
-                     help="Get list of valid story URLs from page given.",)
-   parser.add_option("-n", "--normalize-list",
-                     action="store_true", dest="normalize",default=False,
-                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
-   parser.add_option("-s", "--sites-list",
-                     action="store_true", dest="siteslist",default=False,
-                     help="Get list of valid story URLs examples.",)
-   parser.add_option("-d", "--debug",
-                     action="store_true", dest="debug",
-                     help="Show debug output while downloading.",)
+                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
    
-   (options, args) = parser.parse_args(argv)
+   (options, args) = parser.parse_args()
 
-   if not options.debug:
-       logger = logging.getLogger("fanficdownloader")
-       logger.setLevel(logging.INFO)
-   
-   if not options.siteslist and len(args) != 1:
+   if len(args) != 1:
        parser.error("incorrect number of arguments")
 
-   if options.siteslist:
-       for (site,examples) in adapters.getSiteExamples():
-           print("\n====%s====\n\nExample URLs:"%site)
-           for u in examples:
-               print("  * %s"%u)
-       return
-
    if options.update and options.format != 'epub':
        parser.error("-u/--update-epub only works with epub")
 
-   ## Attempt to update an existing epub.
-   chaptercount = None
-   output_filename = None
-   if options.update:
-       try:
-           (url,chaptercount) = get_dcsource_chaptercount(args[0])
-           if not url:
-               print "No story URL found in epub to update."
-               return
-           print "Updating %s, URL: %s" % (args[0],url)
-           output_filename = args[0]
-       except:
-           # if there's an error reading the update file, maybe it's a URL?
-           # we'll look for an existing outputfile down below.
-           url = args[0]
-   else:
-       url = args[0]
-
-   try:
-       configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
-   except exceptions.UnknownSite, e:
-       if options.list or options.normalize:
-           # list for page doesn't have to be a supported site.
-           configuration = Configuration("test1.com",options.format)
-       else:
-           raise e
+   config = ConfigParser.SafeConfigParser()
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
-
-   if passed_defaultsini:
-       configuration.readfp(passed_defaultsini)
    
    if isfile(join(homepath,"defaults.ini")):
        conflist.append(join(homepath,"defaults.ini"))
    if isfile("defaults.ini"):
        conflist.append("defaults.ini")
        
-   if passed_personalini:
-       configuration.readfp(passed_personalini)
-   
    if isfile(join(homepath,"personal.ini")):
        conflist.append(join(homepath,"personal.ini"))
    if isfile("personal.ini"):
@@ -182,65 +95,42 @@ def main(argv,
        conflist.extend(options.configfile)
       
    logging.debug('reading %s config file(s), if present'%conflist)
-   configuration.read(conflist)
+   config.read(conflist)
 
    try:
-       configuration.add_section("overrides")
+       config.add_section("overrides")
    except ConfigParser.DuplicateSectionError:
        pass
 
    if options.force:
-       configuration.set("overrides","always_overwrite","true")
-
-   if options.update and chaptercount:
-       configuration.set("overrides","output_filename",output_filename)
-       
-   if options.update and not options.updatecover:
-       configuration.set("overrides","never_make_cover","true")
-
-   # images only for epub, even if the user mistakenly turned it
-   # on else where.
-   if options.format not in ("epub","html"):
-       configuration.set("overrides","include_images","false")
+       config.set("overrides","always_overwrite","true")
        
    if options.options:
        for opt in options.options:
            (var,val) = opt.split('=')
-           configuration.set("overrides",var,val)
-
-   if options.list or options.normalize:
-       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
-       print "\n".join(retlist)               
-       return
+           config.set("overrides",var,val)
 
    try:
-       adapter = adapters.getAdapter(configuration,url)
-       adapter.setChaptersRange(options.begin,options.end)
+       ## Attempt to update an existing epub.
+       if options.update:
+           (url,chaptercount) = get_dcsource_chaptercount(args[0])
+           print "Updating %s, URL: %s" % (args[0],url)
+           output_filename = args[0]
+           config.set("overrides","output_filename",args[0])
+       else:
+           url = args[0]
 
-       # check for updating from URL (vs from file)
-       if options.update and not chaptercount:
-           try:
-               writer = writers.getWriter("epub",configuration,adapter)
-               output_filename=writer.getOutputFileName()
-               (noturl,chaptercount) = get_dcsource_chaptercount(output_filename)
-               print "Updating %s, URL: %s" % (output_filename,url)
-           except:
-               options.update = False
-               pass
+       adapter = adapters.getAdapter(config,url,options.format)
 
-       ## Check for include_images without no_image_processing. In absence of PIL, give warning.
-       if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
+       ## Check for include_images and absence of PIL, give warning.
+       if adapter.getConfig('include_images'):
            try:
-               from calibre.utils.magick import Image
-               logging.debug("Using calibre.utils.magick")
+               import Image
            except:
-               try:
-                   import Image
-                   logging.debug("Using PIL")
-               except:
-                   print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
-                   if not sys.stdin.readline().strip().lower().startswith('y'):
-                       return
+               print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
+               if not sys.stdin.readline().strip().lower().startswith('y'):
+                   return
+               
 
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
@@ -263,39 +153,32 @@ def main(argv,
 
        if options.update and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
-
+           
            if chaptercount == urlchaptercount and not options.metaonly:
-               print "%s already contains %d chapters." % (output_filename,chaptercount)
+               print "%s already contains %d chapters." % (args[0],chaptercount)
            elif chaptercount > urlchaptercount:
-               print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
-           elif chaptercount == 0:
-               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
+               print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
            else:
-               # update now handled by pre-populating the old
-               # images and chapters in the adapter rather than
-               # merging epubs.
-               (url,
-                chaptercount,
-                adapter.oldchapters,
-                adapter.oldimgs,
-                adapter.oldcover,
-                adapter.calibrebookmark,
-                adapter.logfile) = get_update_data(output_filename)
-
                print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+               if not options.metaonly:
 
-               if not (options.update and chaptercount == urlchaptercount) \
-                       and adapter.getConfig("do_update_hook"):
-                   chaptercount = adapter.hookForUpdates(chaptercount)
+                   # update now handled by pre-populating the old
+                   # images and chapters in the adapter rather than
+                   # merging epubs.
+                   (url,chaptercount,
+                    adapter.oldchapters,
+                    adapter.oldimgs) = get_update_data(args[0])
 
-               writeStory(configuration,adapter,"epub")
+                   writeStory(config,adapter,"epub")
                    
        else:
            # regular download
            if options.metaonly:
-               pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
+               print adapter.getStoryMetadataOnly()
            
-           output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
+           adapter.setChaptersRange(options.begin,options.end)
+           
+           output_filename=writeStory(config,adapter,options.format,options.metaonly)
        
        if not options.metaonly and adapter.getConfig("post_process_cmd"):
            metadata = adapter.story.metadata
@@ -311,9 +194,9 @@ def main(argv,
        print dne
    except exceptions.UnknownSite, us:
        print us
-
+   
 if __name__ == "__main__":
     #import time
     #start = time.time()
-    main(sys.argv[1:])
+    main()
     #print("Total time seconds:%f"%(time.time()-start))

editconfig.html

@@ -71,7 +71,7 @@
 	Copyright © Fanficdownloader team
       </div>
 
-      <div style="margin-top: 1em; text-align: center">
+      <div style="margin-top: 1em; text-align: center'">
         <script type="text/javascript"><!--
                                           google_ad_client = "pub-2027714004231956";
                                           /* FFD */

example.ini

@@ -1,103 +1,40 @@
 ## This is an example of what your personal configuration might look
-## like.  Uncomment options by removing the '#' in front of them.
+## like.
 
 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  Uncomment by removing '#' in front of is_adult.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Don't like the numbers at the start of chapter titles on some
-## sites?  You can use strip_chapter_numbers to strip them off.  Just
-## want to make them all look the same?  Strip them off, then add them
-## back on with add_chapter_numbers.  Don't like the way it strips
-## numbers or adds them back?  See chapter_title_strip_pattern and
-## chapter_title_add_pattern.
-#strip_chapter_numbers:true
-#add_chapter_numbers:true
-
-[epub]
-## include images from img tags in the body and summary of stories.
-## Images will be converted to jpg for size if possible.  Images work
-## in epub format only.  To get mobi or other format with images,
-## download as epub and use Calibre to convert.
-#include_images:true
-
-## If not set, the summary will have all html stripped for safety.
-## Both this and include_images must be true to get images in the
-## summary.
-#keep_summary_html:true
-
-## If set, the first image found will be made the cover image.  If
-## keep_summary_html is true, any images in summary will be before any
-## in chapters.
-#make_firstimage_cover:true
-
-## Resize images down to width, height, preserving aspect ratio.
-## Nook size, with margin.
-#image_max_size: 580, 725
-
-## Change image to grayscale, if graphics library allows, to save
-## space.
-#grayscale_images: false
-
-
 ## Most common, I expect will be using this to save username/passwords
-## for different sites.  Here are a few examples.  See defaults.ini
-## for the full list.
-
+## for different sites.
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
-## default is false
-#collect_series: true
 
 [www.ficwad.com]
 #username:YourUsername
 #password:YourPassword
 
-[www.twiwrite.net]
-#username:YourName
-#password:yourpassword
-## default is false
-#collect_series: true
-
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.
 #is_adult:true
 
-[www.thewriterscoffeeshop.com]
-#username:YourName
-#password:yourpassword
-#is_adult:true
-## default is false
-#collect_series: true
-
-[www.fictionalley.org]
-#is_adult:true
-
-[www.harrypotterfanfiction.com]
-#is_adult:true
-
-[www.fimfiction.net]
-#is_adult:true
-#fail_on_password: false
-
-[www.tthfanfic.org]
-#is_adult:true
-## tth is a little unusual--it doesn't require user/pass, but the site
-## keeps track of which chapters you've read and won't send another
-## update until it thinks you're up to date.  This way, on download,
-## it thinks you're up to date.
-#username:YourName
-#password:yourpassword
+## The [defaults] section here will override the system [defaults],
+## but not format, site for site:format sections.
+[defaults]
+## Directories only useful in commandline or zip files.
+#output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
+#output_filename: books/${site}/${authorId}/${title}-${storyId}${formatext}
 
+## For example, zip_output here will turn on zip for html and txt, but
+## not epub because the system [epub] section explicitly says
+## zip_output: false (epubs *are* specially formated zip files.)
+#zip_output: true
+#zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 
 ## This section will override anything in the system defaults or other
 ## sections here.
 [overrides]
-## default varies by site.  Set true here to force all sites to
-## collect series.
-#collect_series: true

fanficdownloader/HtmlTagStack.py

@@ -1,57 +0,0 @@
-# coding: utf-8
-
-import re
-import codecs
-
-stack = []
-
-def get_end_tag(tag):
-    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
-        return re.sub(r'.*<([^\ >]+).*', r'</\1>', tag)
-    return u''
-
-def get_tag_name(tag):
-    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
-        return re.sub(r'</*([^\ >]+).*', r'\1', tag)
-    return u''
-
-def push(tag):
-    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
-        stack.append(tag)
-
-def pop():
-    if len(stack) > 0:
-        return stack.pop()
-    return u''
-
-def pop_end_tag():
-    return unicode(get_end_tag(pop()))
-
-def spool_end():
-    html = u''
-    for tag in reversed(stack):
-        html += get_end_tag(tag)
-    return html
-
-def spool_start():
-    html = u''
-    for item in stack:
-        html += item
-    return html
-
-def has_elements():
-    return len(stack) > 0
-
-def get_last():
-    # t = pop()
-    # push(t)
-    # return t
-    if len(stack) > 0:
-        return stack[len(stack)-1]
-    return u''
-
-def flush():
-    del stack[:]
-
-def get_stack():
-    return stack

fanficdownloader/__init__.py

@@ -1,19 +1 @@
 # -*- coding: utf-8 -*-
-
-try:
-    # just a way to switch between web service and CLI/PI
-    import google.appengine.api 
-except:
-    try: # just a way to switch between CLI and PI
-        import calibre.constants
-    except:
-        import sys
-        if sys.version_info >= (2, 7):
-            import logging
-            logger = logging.getLogger(__name__)
-            loghandler=logging.StreamHandler()
-            loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-            logger.addHandler(loghandler)
-            loghandler.setLevel(logging.DEBUG)
-            logger.setLevel(logging.DEBUG)
-

fanficdownloader/adapters/__init__.py

@@ -1,254 +1,106 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob, types
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-logger = logging.getLogger(__name__)
-
-from .. import exceptions as exceptions
-from ..configurable import Configuration
-
-## must import each adapter here.
-
-import adapter_test1
-import adapter_fanfictionnet
-import adapter_fanficcastletvnet
-import adapter_fictionalleyorg
-import adapter_fictionpresscom
-import adapter_ficwadcom
-import adapter_fimfictionnet
-import adapter_harrypotterfanfictioncom
-import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
-import adapter_tenhawkpresentscom
-import adapter_adastrafanficcom
-import adapter_twcslibrarynet
-import adapter_tthfanficorg
-import adapter_twilightednet
-import adapter_twiwritenet
-import adapter_whoficcom
-import adapter_siyecouk
-import adapter_archiveofourownorg
-import adapter_ficbooknet
-import adapter_portkeyorg
-import adapter_mugglenetcom
-import adapter_hpfandomnet
-import adapter_thequidditchpitchorg
-import adapter_nfacommunitycom
-import adapter_midnightwhispersca
-import adapter_ksarchivecom
-import adapter_archiveskyehawkecom
-import adapter_squidgeorgpeja
-import adapter_libraryofmoriacom
-import adapter_wraithbaitcom
-import adapter_checkmatedcom
-import adapter_chaossycophanthexcom
-import adapter_dramioneorg
-import adapter_erosnsapphosycophanthexcom
-import adapter_lumossycophanthexcom
-import adapter_occlumencysycophanthexcom
-import adapter_phoenixsongnet
-import adapter_walkingtheplankorg
-import adapter_ashwindersycophanthexcom
-import adapter_thehexfilesnet
-import adapter_dokugacom
-import adapter_iketernalnet
-import adapter_onedirectionfanfictioncom
-import adapter_storiesofardacom
-import adapter_samdeanarchivenu
-import adapter_destinysgatewaycom
-import adapter_ncisfictionnet
-import adapter_stargateatlantisorg
-import adapter_thealphagatecom
-import adapter_fanfiktionde
-import adapter_ponyfictionarchivenet
-import adapter_sg1heliopoliscom
-import adapter_ncisficcom
-import adapter_nationallibrarynet
-import adapter_themasquenet
-import adapter_pretendercentrecom
-import adapter_darksolaceorg
-import adapter_finestoriescom
-import adapter_hpfanficarchivecom
-import adapter_twilightarchivescom
-import adapter_wizardtalesnet
-import adapter_nhamagicalworldsus
-import adapter_hlfictionnet
-import adapter_grangerenchantedcom
-import adapter_dracoandginnycom
-import adapter_scarvesandcoffeenet
-import adapter_thepetulantpoetesscom
-import adapter_wolverineandroguecom
-import adapter_sinfuldesireorg
-import adapter_merlinficdtwinscouk
-import adapter_thehookupzonenet
-import adapter_bloodtiesfancom
-import adapter_indeathnet
-import adapter_qafficcom
-import adapter_efpfanficnet
-import adapter_potterficscom
-import adapter_efictionestelielde
-import adapter_dotmoonnet
-import adapter_pommedesangcom
-import adapter_restrictedsectionorg
-import adapter_imagineeficcom
-import adapter_buffynfaithnet
-import adapter_psychficcom
-import adapter_hennethannunnet
-import adapter_tokrafandomnetcom
-import adapter_netraptororg
-import adapter_asr3slashzoneorg
-import adapter_nickandgregnet
-import adapter_potterheadsanonymouscom
-import adapter_simplyundeniablecom
-import adapter_scarheadnet
-import adapter_fictionpadcom
-import adapter_storiesonlinenet
-import adapter_trekiverseorg
-import adapter_literotica
-import adapter_voracity2eficcom
-import adapter_spikeluvercom
-import adapter_bloodshedversecom
-import adapter_nocturnallightnet
-import adapter_fanfichu
-import adapter_fanfictioncsodaidokhu
-import adapter_fictionmaniatv
-import adapter_bdsmgeschichten
-import adapter_tolkienfanfiction
-import adapter_themaplebookshelf
-import adapter_fannation
-import adapter_sheppardweircom
-import adapter_samandjacknet
-import adapter_csiforensicscom
-import adapter_lotrfanfictioncom
-import adapter_fhsarchivecom
-import adapter_fanfictionjunkiesde
-
-## This bit of complexity allows adapters to be added by just adding
-## importing.  It eliminates the long if/else clauses we used to need
-## to pick out the adapter.
-
-## List of registered site adapters.
-__class_list = []
-__domain_map = {}
-
-def imports():
-    for name, val in globals().items():
-        if isinstance(val, types.ModuleType):
-            yield val.__name__
-
-for x in imports():
-    if "fanficdownloader.adapters.adapter_" in x:
-        #print x
-        cls = sys.modules[x].getClass()
-        __class_list.append(cls)
-        for site in cls.getAcceptDomains():
-            __domain_map[site]=cls
-
-def getNormalStoryURL(url):
-    r = getNormalStoryURLSite(url)
-    if r:
-        return r[0]
-    else:
-        return None
-
-def getNormalStoryURLSite(url):
-    if not getNormalStoryURL.__dummyconfig:
-        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
-    # pulling up an adapter is pretty low over-head.  If
-    # it fails, it's a bad url.
-    try:
-        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
-        url = adapter.url
-        site = adapter.getSiteDomain()
-        del adapter
-        return (url,site)
-    except:
-        return None
-
-# kludgey function static/singleton
-getNormalStoryURL.__dummyconfig = None
-
-def getAdapter(config,url,anyurl=False):
-
-    #logger.debug("trying url:"+url)
-    (cls,fixedurl) = getClassFor(url)
-    #logger.debug("fixedurl:"+fixedurl)
-    if cls:
-        if anyurl:
-            fixedurl = cls.getSiteExampleURLs().split()[0]
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getConfigSections():
-    return [cls.getConfigSection() for cls in __class_list]
-
-def getSiteExamples():
-    l=[]
-    for cls in sorted(__class_list, key=lambda x : x.getConfigSection()):
-        l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
-    return l
-
-def getConfigSectionFor(url):
-    (cls,fixedurl) = getClassFor(url)
-    if cls:
-        return cls.getConfigSection()
-
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getClassFor(url):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
-    if fixedurl.startswith("//"):
-        fixedurl = "http:%s"%url
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    cls = getClassFromList(domain)
-    if not cls and domain.startswith("www."):
-        domain = domain.replace("www.","")
-        #logger.debug("trying site:without www: "+domain)
-        cls = getClassFromList(domain)
-        fixedurl = re.sub(r"^http(s?)://www\.",r"http\1://",fixedurl)
-    if not cls:
-        #logger.debug("trying site:www."+domain)
-        cls = getClassFromList("www."+domain)
-        fixedurl = re.sub(r"^http(s?)://",r"http\1://www.",fixedurl)
-
-    if cls:
-        fixedurl = cls.stripURLParameters(fixedurl)
-
-    return (cls,fixedurl)
-
-def getClassFromList(domain):
-    try:
-        return __domain_map[domain]
-    except KeyError:
-        pass # return none.
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, re, sys, glob, types
+from os.path import dirname, basename, normpath
+import logging
+import urlparse as up
+
+from .. import exceptions as exceptions
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
+import adapter_archiveofourownorg
+import adapter_ficbooknet
+import adapter_portkeyorg
+import adapter_mugglenetcom
+import adapter_hpfandomnet
+import adapter_thequidditchpitchorg
+import adapter_nfacommunitycom
+import adapter_midnightwhispersca
+import adapter_ksarchivecom
+import adapter_archiveskyehawkecom
+
+## This bit of complexity allows adapters to be added by just adding
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
+    
+## List of registered site adapters.
+__class_list = []
+
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        __class_list.append(sys.modules[x].getClass())
+            
+def getAdapter(config,url,fileform=None):
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+    
+    ## remove any trailing '&' parameters--?sid=999 will be left.
+    ## that's all that any of the current adapters need or want.
+    fixedurl = re.sub(r"&.*$","",fixedurl)
+    
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    logging.debug("site:"+domain)
+    cls = getClassFor(domain)
+    if not cls:
+        logging.debug("trying site:www."+domain)
+        cls = getClassFor("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        adapter.setSectionOrder(adapter.getSiteDomain(),fileform)
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getClassFor(domain):
+    for cls in __class_list:
+        if cls.matchesSite(domain):
+            return cls

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -39,11 +37,12 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
+        self.story.addToList("category","Star Trek")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -53,20 +52,12 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.adastrafanfic.com'
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
-    
     def extractChapterUrlsAndMetadata(self):
 
         if self.is_adult or self.getConfig("is_adult"):
@@ -75,7 +66,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -98,7 +89,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
+        self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php"))
@@ -204,7 +195,6 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -215,7 +205,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -1,4 +1,4 @@
-#  -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 
 # Copyright 2011 Fanficdownloader team
 #
@@ -17,7 +17,6 @@
 
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -31,8 +30,6 @@ def getClass():
     return ArchiveOfOurOwnOrgAdapter
 
 
-logger = logging.getLogger(__name__)
-
 class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -45,25 +42,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                                # utf8) are really windows-1252.
 							   
 							   
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
-        
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-            
-            # normalized story URL.
-            self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ao3')
@@ -75,60 +60,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'archiveofourown.org'
+        return 'www.archiveofourown.org'
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/works/123456 http://"+cls.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+cls.getSiteDomain()+"/works/123456/chapters/78901"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/works/123456"
 
     def getSiteURLPattern(self):
-        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
-        
-    ## Login
-    def needToLoginCheck(self, data):
-        if 'This work is only available to registered users of the Archive.' in data \
-                or "The password or user name you entered doesn't match our records" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url, data):
-
-        params = {}
-        if self.password:
-            params['user_session[login]'] = self.username
-            params['user_session[password]'] = self.password
-        else:
-            params['user_session[login]'] = self.getConfig("username")
-            params['user_session[password]'] = self.getConfig("password")
-        params['user_session[remember_me]'] = '1'
-        params['commit'] = 'Log in'
-        #params['utf8'] = u'✓'#u'\x2713' # gets along with out it, and it confuses the encoder.
-        params['authenticity_token'] = data.split('input name="authenticity_token" type="hidden" value="')[1].split('" /></div>')[0]
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/user_sessions'
-        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['user_session[login]']))
-    
-        d = self._postUrl(loginUrl, params)
-        #logger.info(d)
-    
-        if "Successfully logged in" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['user_session[login]']))
-            raise exceptions.FailedToLogin(url,params['user_session[login]'])
-            return False
-        else:
-            return True
-
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
+        return re.escape("http://"+self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -138,85 +76,57 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         else:
             addurl=""
 
-        metaurl = self.url+addurl
+        meta = self.url+addurl
         url = self.url+'/navigate'+addurl
-        logger.info("url: "+url)
-        logger.info("metaurl: "+metaurl)
+        logging.debug("URL: "+meta)
 
         try:
             data = self._fetchUrl(url)
-            meta = self._fetchUrl(metaurl)
+            meta = self._fetchUrl(meta)
 
             if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
                 raise exceptions.AdultCheckRequired(self.url)
             
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
+                raise exceptions.StoryDoesNotExist(self.meta)
             else:
                 raise e
-
-        if "Sorry, we couldn&#x27;t find the work you were looking for." in data:
-            raise exceptions.StoryDoesNotExist(self.url)
-            
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url,data)
-            data = self._fetchUrl(url,usecache=False)
-            meta = self._fetchUrl(metaurl,usecache=False)
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
-        for tag in soup.findAll('div',id='admin-banner'):
-            tag.extract()
         metasoup = bs.BeautifulSoup(meta)
-        for tag in metasoup.findAll('div',id='admin-banner'):
-            tag.extract()
+        # print data
 
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r"/works/\d+$"))
-        self.story.setMetadata('title',stripHTML(a))
+        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
+        self.story.setMetadata('title',a.string)
 		
         # Find authorid and URL from... author url.
-        alist = soup.findAll('a', href=re.compile(r"/users/\w+/pseuds/\w+"))
-        if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
-            self.story.setMetadata('author','Anonymous')
-            self.story.setMetadata('authorUrl','http://archiveofourown.org/')
-            self.story.setMetadata('authorId','0')
-        else:
-            for a in alist:
-                self.story.addToList('authorId',a['href'].split('/')[-1])
-                self.story.addToList('authorUrl',a['href'])
-                self.story.addToList('author',a.text)
+        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.text)
 
-        newestChapter = None
-        self.newestChapterNum = None # save for comparing during update.
-        # Scan all chapters to find the oldest and newest, on AO3 it's
-        # possible for authors to insert new chapters out-of-order or
-        # change the dates of earlier ones by editing them--That WILL
-        # break epub update.
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
         self.story.setMetadata('numChapters',len(chapters))
-        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        if len(chapters)==1:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
-        else:
-            for index, chapter in enumerate(chapters):
-                # strip just in case there's tags, like <i> in chapter titles.
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        for x in range(0,len(chapters)):
+            # just in case there's tags, like <i> in chapter titles.
+            chapter=chapters[x]
+            if len(chapters)==1:
+                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
+            else:
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-                # (2013-09-21)
-                date = stripHTML(chapter.findNext('span'))[1:-1]
-                chapterDate = makeDate(date,self.dateformat)
-                if newestChapter == None or chapterDate > newestChapter:
-                    newestChapter = chapterDate
-                    self.newestChapterNum = index
+
+
 
         a = metasoup.find('blockquote',{'class':'userstuff'})
         if a != None:
-            self.setDescription(url,a)
+            self.setDescription(url,a.text)
             #self.story.setMetadata('description',a.text)
 		
         a = metasoup.find('dd',{'class':"rating tags"})
@@ -226,28 +136,27 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         a = metasoup.find('dd',{'class':"fandom tags"})
         fandoms = a.findAll('a',{'class':"tag"})
         for fandom in fandoms:
-            self.story.addToList('fandoms',fandom.string)
             self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
         if a != None:
             warnings = a.findAll('a',{'class':"tag"})
             for warning in warnings:
-                self.story.addToList('warnings',warning.string)
+                if warning.string == "Author Chose Not To Use Archive Warnings":
+                    warning.string = "No Archive Warnings Apply"
+                if warning.string != "No Archive Warnings Apply":
+                    self.story.addToList('warnings',warning.string)
 		
         a = metasoup.find('dd',{'class':"freeform tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
-                self.story.addToList('freeformtags',genre.string)
                 self.story.addToList('genre',genre.string)
-                
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
                 if genre != "Gen":
-                    self.story.addToList('ao3categories',genre.string)
                     self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
@@ -255,19 +164,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chars = a.findAll('a',{'class':"tag"})
             for char in chars:
                 self.story.addToList('characters',char.string)
-                
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
-            ships = a.findAll('a',{'class':"tag"})
-            for ship in ships:
-                self.story.addToList('ships',ship.string)
-		
-        a = metasoup.find('dd',{'class':"collections"})
-        if a != None:
-            collections = a.findAll('a')
-            for collection in collections:
-                self.story.addToList('collections',collection.string)
+            chars = a.findAll('a',{'class':"tag"})
+            for char in chars:
+                self.story.addToList('characters',char.string)
 		
+
         stats = metasoup.find('dl',{'class':'stats'})
         dt = stats.findAll('dt')
         dd = stats.findAll('dd')
@@ -278,18 +181,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             if 'Words:' in label:
                 self.story.setMetadata('numWords', value)
 				
-            if 'Comments:' in label:
-                self.story.setMetadata('comments', value)
-				
-            if 'Kudos:' in label:
-                self.story.setMetadata('kudos', value)
-				
-            if 'Hits:' in label:
-                self.story.setMetadata('hits', value)
-				
-            if 'Bookmarks:' in label:
-                self.story.setMetadata('bookmarks', value)
-				
             if 'Chapters:' in label:
                 if value.split('/')[0] == value.split('/')[1]:
                     self.story.setMetadata('status', 'Completed')
@@ -308,58 +199,45 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
 		
-        # Find Series name from series URL.
-        ddseries = metasoup.find('dd',{'class':"series"})
-
-        if ddseries:
-            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
-                series_name = stripHTML(a)
-                series_url = 'http://'+self.host+a['href']
-                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
-                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
-                self.story.setMetadata('series%02dUrl'%i,series_url)
-                if i == 0:
-                    self.setSeries(series_name, series_index)
-                    self.story.setMetadata('seriesUrl',series_url)
-
-    def hookForUpdates(self,chaptercount):
-        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
-            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
-            self.oldchapters = self.oldchapters[:self.newestChapterNum]
-        return len(self.oldchapters)
+        try:
+            # Find Series name from series URL.
+            a = metasoup.find('dd',{'class':"series"})
+            b = a.find('a', href=re.compile(r"/series/\d+"))
+            series_name = b.string
+            series_url = 'http://'+self.host+'/fanfic/'+b['href']
+            series_index = int(a.text.split(' ')[1])
+            self.setSeries(series_name, series_index)
             
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
     # grab the text for an individual chapter.
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 		
-        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-
-        exclude_notes=self.getConfigList('exclude_notes')
-
-        if 'authorheadnotes' not in exclude_notes:
-            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
+		
+        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
+        if headnotes != None:
+            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
             if headnotes != None:
-                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
-                if headnotes != None:
-                    chapter.append("<b>Author's Note:</b>")
-                    chapter.append(headnotes)
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(headnotes)
         
-        if 'chaptersummary' not in exclude_notes:
-            chapsumm = soup.find('div', {'id' : "summary"})
-            if chapsumm != None:
-                chapsumm = chapsumm.find('blockquote')
-                chapter.append("<b>Summary for the Chapter:</b>")
-                chapter.append(chapsumm)
-                
-        if 'chapterheadnotes' not in exclude_notes:
-            chapnotes = soup.find('div', {'id' : "notes"})
+        chapsumm = soup.find('div', {'id' : "summary"})
+        if chapsumm != None:
+            chapsumm = chapsumm.find('blockquote')
+            chapter.append("<b>Summary for the Chapter:</b>")
+            chapter.append(chapsumm)
+        chapnotes = soup.find('div', {'id' : "notes"})
+        if chapnotes != None:
+            chapnotes = chapnotes.find('blockquote')
             if chapnotes != None:
-                chapnotes = chapnotes.find('blockquote')
-                if chapnotes != None:
-                    chapter.append("<b>Notes for the Chapter:</b>")
-                    chapter.append(chapnotes)
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
         chtext = text.find('h3', {'class' : "landmark heading"})
@@ -367,19 +245,17 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chtext.extract()
         chapter.append(text)
 		
-        if 'chapterfootnotes' not in exclude_notes:
-            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-            if chapfoot != None:
-                chapfoot = chapfoot.find('blockquote')
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapfoot)
+        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+        if chapfoot != None:
+            chapfoot = chapfoot.find('blockquote')
+            chapter.append("<b>Notes for the Chapter:</b>")
+            chapter.append(chapfoot)
 		
-        if 'authorfootnotes' not in exclude_notes:
-            footnotes = soup.find('div', {'id' : "work_endnotes"})
-            if footnotes != None:
-                footnotes = footnotes.find('blockquote')
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(footnotes)
+        footnotes = soup.find('div', {'id' : "work_endnotes"})
+        if footnotes != None:
+            footnotes = footnotes.find('blockquote')
+            chapter.append("<b>Author's Note:</b>")
+            chapter.append(footnotes)
 			
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -17,7 +17,6 @@
 
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -49,7 +48,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
@@ -66,22 +65,20 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archive.skyehawke.com'
 
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['archive.skyehawke.com','www.skyehawke.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://")+r"(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
+
+
+        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -91,6 +88,8 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
+        data = self._fetchUrl(url)
+            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data
@@ -99,7 +98,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=stripHTML(a).split('"')[1]
+        title=a.text.split('"')[1]
         self.story.setMetadata('title',title)
         
         # Find authorid and URL from... author url.
@@ -152,7 +151,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
                 rating.find('br').replaceWith('split')
                 rating=rating.text.replace("This story is rated",'').split('split')[0]
                 self.story.setMetadata('rating',rating)
-                logger.debug(self.story.getMetadata('rating'))
+                logging.debug(self.story.getMetadata('rating'))
 			
                 warnings=box.find('ol')
                 if warnings != None:
@@ -178,7 +177,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -1,254 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return AshwinderSycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','asph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'ashwinder.sycophanthex.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'This story contains adult content and/or themes.' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Logout" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                self.story.setMetadata('title',stripHTML(a))
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'table':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = info.findAll('b')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word Count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if genre != 'none':
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if warning != ' none':
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"align" : "left"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -1,227 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return Asr3SlashzoneOrgAdapter
-
-class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','asr3')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'asr3.slashzone.org'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=3"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        #print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Rating
-        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
-        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
-        self.story.setMetadata('rating', rate)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        metadiv = soup.find('div',{'class':'content'})
-        smalldiv = metadiv.find('div',{'class':'small'})
-
-        categorys = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for category in categorys:
-            self.story.addToList('category',category.string)
-        
-        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-
-        ships = smalldiv.parent.findAll('a',href=re.compile(r'browse\.php\?type=class&type_id=2&classid=1'))
-        for ship in ships:
-            self.story.addToList('ships',ship.string)
-
-        metatext = stripHTML(smalldiv)
-
-        if 'Completed: Yes' in metatext:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        wordstart=metatext.rindex('Word count:')+12
-        words = metatext[wordstart:metatext.index(' ',wordstart)]
-        self.story.setMetadata('numWords', words)
-
-        datesdiv = soup.find('div',{'class':'bottom'})
-        dates = stripHTML(datesdiv).split()
-        # Published: 04/26/2011 Updated: 03/06/2013
-        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/archive/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-        # remove 'small' leaving only summary.
-        smalldiv.extract()
-        self.setDescription(url,metadiv)
-        
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_bdsmgeschichten.py

@@ -1,346 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2014 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import urlparse
-import time
-
-from .. import BeautifulSoup as bs
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter, makeDate
-
-def _translate_date_german_english(date):
-    fullmon = {"Januar":"01",
-               "Februar":"02",
-               u"März":"03",
-               "April":"04",
-               "Mai":"05",
-               "Juni":"06",
-               "Juli":"07",
-               "August":"08",
-               "September":"09",
-               "Oktober":"10",
-               "November":"11",
-               "Dezember":"12"}
-    for (name,num) in fullmon.items():
-        date = date.replace(name,num)
-    return date
-
-_REGEX_TRAILING_DIGIT = re.compile("(\d+)$")
-_REGEX_DASH_TO_END = re.compile("-[^-]+$")
-_REGEX_CHAPTER_TITLE = re.compile(ur"""
-    \s*
-    [\u2013-]?
-    \s*
-    ([\dIVX-]+)?
-    \.?
-    \s*
-    [\[\(]?
-    \s*
-    (Teil|Kapitel|Tag)?
-    \s*
-    ([\dIVX-]+)?
-    \s*
-    [\]\)]?
-    \s*
-    $
-""", re.VERBOSE)
-_INITIAL_STEP = 5
-
-class BdsmGeschichtenAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8", "Windows-1252"]
-
-        self.story.setMetadata('siteabbrev','bdsmgesch')
-
-        # Replace possible chapter numbering
-        chapterMatch = _REGEX_TRAILING_DIGIT.search(url)
-        if chapterMatch is None:
-            self.maxChapter = 1
-        else:
-            self.maxChapter = int(chapterMatch.group(1))
-        # url = re.sub(_REGEX_TRAILING_DIGIT, "1", url)
-
-        # set storyId
-        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
-
-        # normalize URL
-        self._setURL('http://%s/%s' % (self.getSiteDomain(), self.story.getMetadata('storyId')))
-
-        self.dateformat = '%d. %m %Y - %H:%M'
-
-    @staticmethod
-    def getSiteDomain():
-        return 'bdsm-geschichten.net'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.bdsm-geschichten.net', 'www.bdsm-geschichten.net']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://www.bdsm-geschichten.net/title-of-story-1 http://bdsm-geschichten.net/title-of-story-1"
-
-    def getSiteURLPattern(self):
-        return r"http://(www\.)?bdsm-geschichten.net/(?P<storyId>[a-zA-Z0-9_-]+)"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if not (self.is_adult or self.getConfig("is_adult")):
-            raise exceptions.AdultCheckRequired(self.url)
-
-        try:
-            data1 = self._fetchUrl(self.url)
-            soup = bs.BeautifulSoup(data1)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        #strip comments from soup
-        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
-
-        # Cache the soups so we won't have to redownload in getChapterText later
-        self.soupsCache = {}
-        self.soupsCache[self.url] = soup
-
-        # author
-        authorDiv = soup.find("div", "author-pane-line author-name")
-        authorId = authorDiv.string.strip()
-        self.story.setMetadata('authorId', authorId)
-        self.story.setMetadata('author', authorId)
-        # TODO not really true need to be loggedin for this to work or fetch userid
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId)
-
-        # TODO better metadata
-        date = soup.find("div", {"class": "submitted"}).string.strip()
-        date = re.sub(" &#151;.*", "", date)
-        date = _translate_date_german_english(date)
-        self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-        title1 = soup.find("h1", {'class': 'title'}).string
-
-
-        for tagLink in soup.find("ul", "taxonomy").findAll("a"):
-            self.story.addToList('category', tagLink.string)
-
-        ## Retrieve chapter soups
-        if self.getConfig('find_chapters') == 'guess':
-            self.chapterUrls = []
-            self._find_chapters_by_guessing(title1)
-        else:
-            self._find_chapters_by_parsing(soup)
-
-        firstChapterUrl = self.chapterUrls[0][1]
-        if firstChapterUrl in self.soupsCache:
-            firstChapterSoup = self.soupsCache[firstChapterUrl]
-            h1 = firstChapterSoup.find("h1").text
-        else:
-            h1 = soup.find("h1").text
-
-        h1 = re.sub(_REGEX_CHAPTER_TITLE, "", h1)
-        self.story.setMetadata('title', h1)
-        self.story.setMetadata('numChapters', len(self.chapterUrls))
-        return
-
-    def _find_chapters_by_parsing(self, soup):
-
-        # store original soup
-        origSoup = soup
-
-        #
-        # find first chapter
-        #
-        firstLink = None
-        firstLinkDiv = soup.find("div", "field-field-erster-teil")
-        if firstLinkDiv is not None:
-            firstLink = "http://%s%s" % (self.getSiteDomain(), firstLinkDiv.findNext("a")['href'])
-            logger.debug("Found first chapter right away <%s>" % firstLink)
-            try:
-                soup = bs.BeautifulSoup(self._fetchUrl(firstLink))
-                self.soupsCache[firstLink] = soup
-                self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(self.url)
-                else:
-                    raise exceptions.StoryDoesNotExist(firstLink)
-        else:
-            logger.debug("DIDN'T find first chapter right away")
-            # parse previous Link until first
-            while True:
-                prevLink = None
-                prevLinkDiv = soup.find("div", "field-field-vorheriger-teil")
-                if prevLinkDiv is not None:
-                    prevLink = prevLinkDiv.find("a")
-                if prevLink is None:
-                    prevLink = soup.find("a", text=re.compile("&lt;&lt;&lt;")) # <<<
-                if prevLink is None:
-                    logger.debug("Couldn't find prev part")
-                    break
-                else:
-                    logger.debug("Previous Chapter <%s>" % prevLink)
-                    if type(prevLink) != bs.Tag or prevLink.name != "a":
-                        prevLink = prevLink.findParent("a")
-                    if prevLink is None or '#' in prevLink['href']:
-                        logger.debug("Couldn't find prev part (false positive) <%s>" % prevLink)
-                        break
-                    prevLink = prevLink['href']
-                try:
-                    soup = bs.BeautifulSoup(self._fetchUrl(prevLink))
-                    self.soupsCache[prevLink] = soup
-                    prevTtitle = soup.find("h1", {'class': 'title'}).string
-                    self.chapterUrls.insert(0, (prevTtitle, prevLink))
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(nextLink)
-                    else:
-                        raise e
-                firstLink = prevLink
-
-        # if first chapter couldn't be determined, assume the URL originally
-        # passed is the first chapter
-        if firstLink is None:
-            logger.debug("Couldn't set first chapter")
-            firstLink = self.url
-            self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
-
-        # set first URL
-        logger.debug("Set first link: %s" % firstLink)
-        self._setURL(firstLink)
-        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(firstLink).group('storyId'))
-
-        #
-        # Parse next chapters
-        #
-        while True:
-            nextLink = None
-            nextLinkDiv = soup.find("div", "field-field-naechster-teil")
-            if nextLinkDiv is not None:
-                nextLink = nextLinkDiv.find("a")
-            if nextLink is None:
-                nextLink = soup.find("a", text=re.compile("&gt;&gt;&gt;"))
-            if nextLink is None:
-                nextLink = soup.find("a", text=re.compile("Fortsetzung"))
-
-            if nextLink is None:
-                logger.debug("Couldn't find next part")
-                break
-            else:
-                if type(nextLink) != bs.Tag or nextLink.name != "a":
-                    nextLink = nextLink.findParent("a")
-                if nextLink is None or '#' in nextLink['href']:
-                    logger.debug("Couldn't find next part (false positive) <%s>" % nextLink)
-                    break
-                nextLink = nextLink['href']
-
-            if not nextLink.startswith('http:'):
-                nextLink = 'http://' + self.getSiteDomain() + nextLink
-
-            for loadedChapter in self.chapterUrls:
-                if loadedChapter[0] == nextLink:
-                    logger.debug("ERROR: Repeating chapter <%s> Try to fix it" % nextLink)
-                    nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink)
-                    if nextLinkMatch is not None:
-                        curChap = nextLinkMatch.group(1)
-                        nextLink = re.sub(_REGEX_TRAILING_DIGIT, str(int(curChap) + 1), nextLink)
-                    else:
-                        break
-            try:
-                data = self._fetchUrl(nextLink)
-                soup = bs.BeautifulSoup(data)
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(nextLink)
-                else:
-                    raise e
-            title2 = soup.find("h1", {'class': 'title'}).string
-            self.chapterUrls.append((title2, nextLink))
-            logger.debug("Grabbing next chapter URL " + nextLink)
-            self.soupsCache[nextLink] = soup
-            # [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
-        logger.debug("Chapters: %s" % self.chapterUrls)
-
-
-    def _find_chapters_by_guessing(self, title1):
-        step = _INITIAL_STEP
-        curMax = self.maxChapter + step
-        lastHit = True
-        while True:
-            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(curMax), self.url) 
-            if nextChapterUrl == self.url:
-                logger.debug("Unable to guess next chapter because URL doesn't end in numbers")
-                break;
-            try:
-                logger.debug("Trying chapter URL " + nextChapterUrl)
-                data = self._fetchUrl(nextChapterUrl)
-                hit = True
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    hit = False
-                else:
-                    raise e
-            if hit:
-                logger.debug("Found chapter URL " + nextChapterUrl)
-                self.maxChapter = curMax
-                self.soupsCache[nextChapterUrl] = bs.BeautifulSoup(data)
-                if not lastHit:
-                    break
-                lastHit = curMax
-                curMax += step
-            else:
-                lastHit = False
-                curMax -= 1
-            logger.debug(curMax)
-
-        for i in xrange(1, self.maxChapter):
-            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(i), self.url)
-            nextChapterTitle = re.sub("1", str(i), title1)
-            self.chapterUrls.append((nextChapterTitle, nextChapterUrl))
-
-    def getChapterText(self, url):
-
-        if url in self.soupsCache:
-            logger.debug('Getting chapter <%s> from cache' % url)
-            soup = self.soupsCache[url]
-        else:
-            logger.debug('Downloading chapter <%s>' % url)
-            data1 = self._fetchUrl(url)
-            soup = bs.BeautifulSoup(data1)
-            #strip comments from soup
-            [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
-
-        # get story text
-        storyDiv1 = bs.Tag(soup, "div")
-        for para in soup.find("div", "full-node").find('div', 'content').findAll("p"):
-            storyDiv1.append(para)
-        storyDiv1.append('<br />')
-        storytext = self.utf8FromSoup(url,storyDiv1)
-
-        return storytext
-
-
-def getClass():
-    return BdsmGeschichtenAdapter

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -1,193 +0,0 @@
-from datetime import timedelta
-import re
-import urllib2
-import urlparse
-
-from .. import BeautifulSoup
-from ..htmlcleanup import stripHTML
-
-from base_adapter import BaseSiteAdapter, makeDate
-from .. import exceptions
-
-
-def getClass():
-    return BloodshedverseComAdapter
-
-
-def _get_query_data(url):
-    components = urlparse.urlparse(url)
-    query_data = urlparse.parse_qs(components.query)
-    return dict((key, data[0]) for key, data in query_data.items())
-
-
-class BloodshedverseComAdapter(BaseSiteAdapter):
-    SITE_ABBREVIATION = 'bvc'
-    SITE_DOMAIN = 'bloodshedverse.com'
-
-    BASE_URL = 'http://' + SITE_DOMAIN + '/'
-    READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
-
-    STARTED_DATETIME_FORMAT = '%m/%d/%Y'
-    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        query_data = urlparse.parse_qs(self.parsedUrl.query)
-        story_no = query_data['no'][0]
-
-        self.story.setMetadata('storyId', story_no)
-        self._setURL(self.READ_URL_TEMPLATE % story_no)
-        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
-
-    def _customized_fetch_url(self, url, exception=None, parameters=None):
-        if exception:
-            try:
-                data = self._fetchUrl(url, parameters)
-            except urllib2.HTTPError:
-                raise exception(self.url)
-        # Just let self._fetchUrl throw the exception, don't catch and
-        # customize it.
-        else:
-            data = self._fetchUrl(url, parameters)
-
-        return BeautifulSoup.BeautifulSoup(data)
-
-    @staticmethod
-    def getSiteDomain():
-        return BloodshedverseComAdapter.SITE_DOMAIN
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return cls.READ_URL_TEMPLATE % 1234
-
-    def getSiteURLPattern(self):
-        return re.escape(self.BASE_URL + 'stories.php?go=') + r'(read|chapters)\&no=\d+$'
-
-    # Override stripURLParameters so the "no" parameter won't get stripped
-    @classmethod
-    def stripURLParameters(cls, url):
-        return url
-
-    def extractChapterUrlsAndMetadata(self):
-        soup = self._customized_fetch_url(self.url)
-
-        # Since no 404 error code we have to raise the exception ourselves.
-        # A title that is just 'by' indicates that there is no author name
-        # and no story title available.
-        if stripHTML(soup.title) == 'by':
-            raise exceptions.StoryDoesNotExist(self.url)
-
-        for option in soup.find('select', {'name': 'chapter'}):
-            title = stripHTML(option)
-            url = self.READ_URL_TEMPLATE % option['value']
-            self.chapterUrls.append((title, url))
-
-        # Get the URL to the author's page and find the correct story entry to
-        # scrape the metadata
-        author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
-        soup = self._customized_fetch_url(author_url)
-
-        story_no = self.story.getMetadata('storyId')
-        # Ignore first list_box div, it only contains the author information
-        for list_box in soup('div', {'class': 'list_box'})[1:]:
-            url = list_box.find('a', {'class': 'fictitle'})['href']
-            query_data = _get_query_data(url)
-
-            # Found the div containing the story's metadata; break the loop and
-            # parse the element
-            if query_data['no'] == story_no:
-                break
-        else:
-            raise exceptions.FailedToDownload(self.url)
-
-        title_anchor = list_box.find('a', {'class': 'fictitle'})
-        self.story.setMetadata('title', stripHTML(title_anchor))
-
-        author_anchor = title_anchor.findNextSibling('a')
-        self.story.setMetadata('author', stripHTML(author_anchor))
-        self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
-        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
-
-        list_review = list_box.find('div', {'class': 'list_review'})
-        reviews = stripHTML(list_review.a).split(' ', 1)[0]
-        self.story.setMetadata('reviews', reviews)
-
-        summary_div = list_box.find('div', {'class': 'list_summary'})
-        if not self.getConfig('keep_summary_html'):
-            summary = ''.join(summary_div(text=True))
-        else:
-            summary = self.utf8FromSoup(author_url, summary_div)
-
-        self.story.setMetadata('description', summary)
-
-        # I'm assuming this to be the category, not sure what else it could be
-        first_listinfo = list_box.find('div', {'class': 'list_info'})
-        self.story.addToList('category', stripHTML(first_listinfo.a))
-
-        for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
-            for b_tag in list_info('b'):
-                key = b_tag.string.strip(': ')
-                # Strip colons from the beginning, superfluous spaces and minus
-                # characters from the end, and possibly trailing commas from
-                # the warnings if only one is present
-                value = b_tag.nextSibling.string.strip(': -,')
-
-                if key == 'Genre':
-                    for genre in value.split(', '):
-                        # Ignore the "none" genre
-                        if not genre == 'none':
-                            self.story.addToList('genre', genre)
-
-                elif key == 'Rating':
-                    self.story.setMetadata('rating', value)
-
-                elif key == 'Complete':
-                    self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
-
-                elif key == 'Warning':
-                    for warning in value.split(', '):
-                        # The string here starts with ", " before the actual list
-                        # of values sometimes, so check for an empty warning
-                        # and ignore the "none" warning.
-                        if not warning or warning == 'none':
-                            continue
-
-                        self.story.addToList('warnings', warning)
-
-                elif key == 'Chapters':
-                    self.story.setMetadata('numChapters', int(value))
-
-                elif key == 'Words':
-                    # Apparently only numChapters need to be an integer for
-                    # some strange reason. Remove possible ',' characters as to
-                    # not confuse the codebase down the line
-                    self.story.setMetadata('numWords', value.replace(',', ''))
-
-                elif key == 'Started':
-                    self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
-
-                elif key == 'Updated':
-                    date_string, period = value.rsplit(' ', 1)
-                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
-
-                    # Rather ugly hack to work around Calibre's changing of
-                    # Python's locale setting, causing am/pm to not be properly
-                    # parsed by strptime() when using a non-english locale
-                    if period == 'pm':
-                        date += timedelta(hours=12)
-                    self.story.setMetadata('dateUpdated', date)
-
-        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
-            raise exceptions.AdultCheckRequired(self.url)
-
-    def getChapterText(self, url):
-        soup = self._customized_fetch_url(url)
-        storytext_div = soup.find('div', {'class': 'storytext'})
-
-        if self.getConfig('strip_text_links'):
-            for anchor in storytext_div('a', {'class': 'FAtxtL'}):
-                navigable_string = BeautifulSoup.NavigableString(anchor.string)
-                anchor.replaceWith(navigable_string)
-
-        return self.utf8FromSoup(url, storytext_div)

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -1,337 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# By virtue of being recent and requiring both is_adult and user/pass,
-# adapter_fanficcastletvnet.py is the best choice for learning to
-# write adapters--especially for sites that use the eFiction system.
-# Most sites that have ".../viewstory.php?sid=123" in the story URL
-# are eFiction.
-
-# For non-eFiction sites, it can be considerably more complex, but
-# this is still a good starting point.
-
-# In general an 'adapter' needs to do these five things:
-
-# - 'Register' correctly with the downloader
-# - Site Login (if needed)
-# - 'Are you adult?' check (if needed--some do one, some the other, some both)
-# - Grab the chapter list
-# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
-# - Grab the chapter texts
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return BloodTiesFansComAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','btf') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'bloodties-fans.com' # XXX
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-
-            # Furthermore, there's a couple sites now with more than
-            # one warning level for different ratings.  And they're
-            # fussy about it.  midnightwhispers has three: 4, 2 & 1.
-            # we'll try 1 first.
-            addurl = "&ageconsent=ok&warning=4" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  nfacommunity uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=561&warning=4
-        # viewstory.php?sid=561&warning=1
-        # viewstory.php?sid=561&warning=2
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/fiction/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/fiction/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -1,296 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import cookielib as cl
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return BuffyNFaithNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class BuffyNFaithNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        
-        self.setHeader()
-        
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-            
-            # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
-            nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
-            self._setURL(nurl)
-            #argh, this mangles the ampersands I need on metadata['storyUrl']
-            #will set it this way
-            self.story.setMetadata('storyUrl',nurl,condremoveentities=False)
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-        
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','bnfnet')
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'buffynfaith.net'
-
-    @classmethod
-    def stripURLParameters(cls,url):
-        "Only needs to be overriden if URL contains more than one parameter"
-        ## This adapter needs at least two parameters left on the URL, act and id
-        return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url)
-    
-    def setHeader(self):
-        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
-        
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=ovr&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234&ch=2"
-
-    def getSiteURLPattern(self):
-        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
-        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949
-        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2
-        p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\
-            r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
-        return p
-
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
-    
-    def extractChapterUrlsAndMetadata(self):
-
-        dateformat = "%d %B %Y"
-        url = self.url
-        logger.debug("URL: "+url)
-        
-        #set a cookie to get past adult check
-        if self.is_adult or self.getConfig("is_adult"):
-            cookie = cl.Cookie(version=0, name='my_age', value='yes',
-                               port=None, port_specified=False,
-                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
-                               path='/', path_specified=True,
-                               secure=False,
-                               expires=time.time()+10000,
-                               discard=False,
-                               comment=None,
-                               comment_url=None,
-                               rest={'HttpOnly': None},
-                               rfc2109=False)
-            self.cookiejar.set_cookie(cookie)
-            self.setHeader()
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        #print data
-
-        if "ADULT CONTENT WARNING" in data: 
-            raise exceptions.AdultCheckRequired(self.url)
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        #stuff in <head>: description
-        svalue = soup.head.find('meta',attrs={'name':'description'})['content']
-        #self.story.setMetadata('description',svalue)
-        self.setDescription(url,svalue)
-        
-        #useful stuff in rest of doc, all contained in this:
-        doc = soup.body.find('div', id='my_wrapper')
-        
-        #first the site category (more of a genre to me, meh) and title, in this element:
-        mt = doc.find('div',attrs={'class':'maintitle'})
-        self.story.addToList('genre',mt.findAll('a')[1].string)
-        self.story.setMetadata('title',mt.findAll('a')[1].nextSibling[len('&nbsp;&raquo;&nbsp;'):])
-        del mt
-
-        #the actual category, for me, is 'Buffy: The Vampire Slayer'
-        #self.story.addToList('category','Buffy: The Vampire Slayer')
-        #No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini
-        
-        #then a block that sits in a table cell like so:
-        #(contains a lot of metadata)
-        mblock = doc.find('td', align='left', width = '70%').contents
-        while len(mblock) > 0:
-            i = mblock.pop(0)
-            if 'Author:' in i.string:
-                #drop empty space
-                mblock.pop(0)
-                #get author link
-                a = mblock.pop(0)
-                authre = re.escape('./index.php?act=bio&id=')+'(?P<authid>\d+)'
-                m = re.match(authre,a['href'])
-                self.story.setMetadata('author',a.string)
-                self.story.setMetadata('authorId',m.group('authid'))
-                authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('authorId'))
-                self.story.setMetadata('authorUrl',authurl,condremoveentities=False)
-                #drop empty space
-                mblock.pop(0)
-            if 'Rating:' in i.string:
-                self.story.setMetadata('rating',mblock.pop(0).strip())
-            if 'Published:' in i.string:
-                date = mblock.pop(0).strip()
-                #get rid of 'st', 'nd', 'rd', 'th' after day number
-                date = date[0:2]+date[4:]
-                self.story.setMetadata('datePublished',makeDate(date, dateformat))
-            if 'Last Updated:' in i.string:
-                date = mblock.pop(0).strip()
-                #get rid of 'st', 'nd', 'rd', 'th' after day number
-                date = date[0:2]+date[4:]
-                self.story.setMetadata('dateUpdated',makeDate(date, dateformat))
-            if 'Genre:' in i.string:
-                genres = mblock.pop(0).strip()
-                genres = genres.split('/')
-                for genre in genres: self.story.addToList('genre',genre)
-            #end ifs
-        #end while
-
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'ch' } )
-         
-        if select is None:
-           # no selector found, so it's a one-chapter story.
-           #self.chapterUrls.append((self.story.getMetadata('title'),url))
-           self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            o['value'])
-                title = u"%s" % o
-                title = stripHTML(title)
-                ts = title.split(' ',1)
-                title = ts[0]+'. '+ts[1]
-                self.chapterUrls.append((title,url))
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        ## Go scrape the rest of the metadata from the author's page.
-        data = self._fetchUrl(self.story.getMetadata('authorUrl'))
-        soup = bs.BeautifulSoup(data)
-        #find the story link and its parent div
-        storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
-        storydiv = storya.parent
-        #warnings come under a <spawn> tag. Never seen that before...
-        #appears to just be a line of freeform text, not necessarily a list
-        #optional
-        spawn = storydiv.find('spawn',{'id':'warnings'})
-        if spawn is not None:
-            warns = spawn.nextSibling.strip()
-            self.story.addToList('warnings',warns)
-        #some meta in spans - this should get all, even the ones jammed in a table
-        spans = storydiv.findAll('span')
-        for s in spans:
-            if s.string == 'Ship:':
-                list = s.nextSibling.strip().split()
-                self.story.extendList('ships',list)
-            if s.string == 'Characters:':
-                list = s.nextSibling.strip().split(',')
-                self.story.extendList('characters',list)
-            if s.string == 'Status:':
-                st = s.nextSibling.strip()
-                self.story.setMetadata('status',st)
-            if s.string == 'Words:':
-                st = s.nextSibling.strip()
-                self.story.setMetadata('numWords',st)
-
-        #reviews - is this worth having?
-        #ffnet adapter gathers it, don't know if anything else does
-        #or if it's ever going to be used!
-        a = storydiv.find('a',{'id':'bold-blue'})
-        if a:
-            revs = a.nextSibling.strip()[1:-1]
-            self.story.setMetadata('reviews',st)
-        else:
-            revs = '0'
-            self.story.setMetadata('reviews',st)
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'fanfiction'})
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-        
-        #remove all the unnecessary bookmark tags
-        [s.extract() for s in div('div',{'class':"tiny_box2"})]
-        
-        #is there a review link?
-        r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$"))
-        if r is not None:
-        #remove the review link and its parent div
-            r.parent.extract()
-        
-        #There might also be a link to the sequel on the last chapter
-        #I'm inclined to keep it in, but the URL needs to be changed from relative to absolute
-        #Shame there isn't proper series metadata available
-        #(I couldn't find it anyway)
-        s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$"))
-        if s is not None:
-            s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:]
-
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -53,11 +51,11 @@ from base_adapter import BaseSiteAdapter,  makeDate
 # updated to reflect the class below it.  That, plus getSiteDomain()
 # take care of 'Registering'.
 def getClass():
-    return TheHookupZoneNetAdapter # XXX
+    return CastleFansOrgAdapter # XXX
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
+class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -73,14 +71,19 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/CriminalMinds/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/fanfic/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','thupz') # XXX
+        self.story.setMetadata('siteabbrev','cslf') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Castle") # XXX
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
@@ -89,14 +92,13 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'thehookupzone.net' # XXX
+        return 'castlefans.org' # XXX
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -119,14 +121,14 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         params['cookiecheck'] = '1'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/CriminalMinds/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -148,7 +150,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -180,18 +182,18 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
+        self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/CriminalMinds/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/CriminalMinds/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfic/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -276,7 +278,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/CriminalMinds/'+a['href']
+            series_url = 'http://'+self.host+'/fanfic/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -285,7 +287,6 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -296,7 +297,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -1,238 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ChaosSycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ChaosSycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','csph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'chaos.sycophanthex.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=19"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Age Consent Required" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        rating=pt.text.split('(')[1].split(')')[0]
-        self.story.setMetadata('rating', rating)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-				
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-		
-        labels = soup.findAll('span',{'class':'label'})
-		
-        value = labels[0].previousSibling
-        svalue = ""
-        while value != None:
-            val = value
-            value = value.previousSibling
-        while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-		
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value.split(' -')[0])
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Complete' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -1,238 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return CheckmatedComAdapter
-
-
-class CheckmatedComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-
-        self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','chm')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.checkmated.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/story.php?story=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?story=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites. This story is in The Bedchamber
-    def needToLoginCheck(self, data):
-        if 'This story is in The Bedchamber' in data \
-                or 'That username is not in our database' in data \
-                or "That password is not correct, please try again" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['name'] = self.username
-            params['pass'] = self.password
-        else:
-            params['name'] = self.getConfig("username")
-            params['pass'] = self.getConfig("password")
-        params['login'] = 'yes'
-        params['submit'] = 'login'
-
-        loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
-        d = self._fetchUrl(loginUrl,params)
-        e = self._fetchUrl(url)
-
-        if "Welcome back," not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['name']))
-            raise exceptions.FailedToLogin(url,params['name'])
-            return False
-        elif "This story is in The Bedchamber" in e:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Your account does not have sufficient priviliges to read this story.")
-            return False
-        else:
-            return True
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        ## Title
-        a = soup.find('span', {'class' : 'storytitle'})
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = a.parent.find('a', href=re.compile(r"authors.php\?name\=\w+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        a = soup.find('select', {'name' : 'chapter'})
-        if a == None:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in a.findAll('option'):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-
-        # website does not keep track of word count, and there is no convenient way to calculate it
-
-        summary = soup.find('fieldset')
-        summary.find('legend').extract()
-        summary.name='div'
-        self.setDescription(url,summary)
-
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        table = soup.findAll('div', {'class' : 'text'})[1]
-        for labels in table.findAll('tr'):
-            value = labels.findAll('td')[1]
-            label = labels.findAll('td')[0]
-
-
-            if 'Rating' in stripHTML(label):
-                self.story.setMetadata('rating', stripHTML(value))
-
-            if 'Ship' in stripHTML(label):
-                if value.string != "none/none":
-                    self.story.addToList('ships',value.string)
-                for char in value.string.split('/'):
-                    if char != 'none':
-                        self.story.addToList('characters',char)
-
-            if 'Status' in stripHTML(label):
-                if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in stripHTML(label):
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in stripHTML(label):
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')
-        for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
-            a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
-            if a != None:
-                for labels in story.findAll('tr'):
-                    value = labels.findAll('td')[1]
-                    label = labels.findAll('td')[0]
-                    if 'genre' in stripHTML(label):
-                        for genre in value.findAll('img'):
-                            self.story.addToList('genre',genre['title'])
-
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        div = soup.find('div', {'id' : 'resizeableText'})
-        div.find('div', {'class' : 'storyTools'}).extract()
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_csiforensicscom.py

@@ -1,237 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return CSIForensicsComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class CSIForensicsComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','csiforensics')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'csi-forensics.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=5&skin=elegantcsi"
-        else:
-            addurl="&skin=elegantcsi"
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-				
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "This story is rated NC-17, and therefore is not suitable for minors.  If you are below the age required to view such material in your locality, please return from whence you came." in data: # XXX 
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-       
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-       
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Rating
-        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
-        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
-        self.story.setMetadata('rating', rate)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-        
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        smalldiv = soup.find('div', {'class' : 'small'})
-        
-      
-        chars = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-            
-        metatext = stripHTML(smalldiv)
-
-        if 'Completed: Yes' in metatext:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress') 
-        
-        word=soup.find(text=re.compile("Word count:")).split(':')
-        self.story.setMetadata('numWords', word[1])
-
-        cats = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-            
-        warnings = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=class(&amp;)type_id=2(&amp;)classid=\d+'))
-        for warning in warnings:
-            self.story.addToList('warnings',warning.string)            
-            			
-        date=soup.find('div',{'class' : 'bottom'})
-        pd=date.find(text=re.compile("Published:")).string.split(': ')
-        self.story.setMetadata('datePublished', makeDate(stripHTML(pd[1].split(' U')[0]), self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(stripHTML(pd[2]), self.dateformat))
-		
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        pub=0
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Genres' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:                
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-        smalldiv.extract()
-        
-        # Summary  
-        summary = soup.find('div', {'class' : 'content'})
-        self.setDescription(url,summary)        
-           
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)
- 

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -1,336 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DarkSolaceOrgAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DarkSolaceOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dksl')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'dark-solace.org'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.dark-solace.org','dark-solace.org']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data \
-                or "Registered Users Only" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['action'] = 'login'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._postUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #User Account Page
-            logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        div = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = div.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-
-        # first a tag in pagetitle is title
-        self.story.setMetadata('title',stripHTML(div.find('a')))
-        div.find('a').extract()
-        # only thing left in div(pagetitle) now should be 'by' and rating.
-        rating = stripHTML(div)
-        if '[' in rating:
-            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
-
-        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
-                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
-            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
-        
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                    self.story.getMetadata('storyId')+'($|[^\d])'))
-        # author's story list is paginated if there's a pagelinks div.
-        # Only need to look in it if the story wasn't on the first page.
-        pagelinks = asoup.find('div',{'id':'pagelinks'})
-        if pagelinks and storylink==None:
-            authpageslist = pagelinks.findAll('a',href=re.compile(r'action=storiesby'))
-            for page in authpageslist[1:]: # skip first, already checked above.
-                asoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
-                storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                            self.story.getMetadata('storyId')+'($|[^\d])'))
-                if storylink:
-                    break
-
-        if not storylink:
-            raise exceptions.FailedToDownload("Unable to find story metadata on author's page(s)")
-        
-        metalist = storylink.parent.parent
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = metalist.findAll('span', {'class' : 'label'})
-        for labelspan in labels:
-            label = labelspan.text
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while value and not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = metalist.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/elysian/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                             self.story.getMetadata('storyId')+'($|[^\d])'))
-            if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
-                storylink = None
-
-            offset = 0
-            # series story list is paginated if there's a pagelinks div.
-            # Only need to look in it if the story wasn't on the first page.
-            pagelinks = seriessoup.find('div',{'id':'pagelinks'})
-            if pagelinks and storylink==None:
-                authpageslist = pagelinks.findAll('a',href=re.compile(r'offset='))
-                for page in authpageslist[1:]: # skip first, already checked above.
-                    seriessoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
-                    storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                                     self.story.getMetadata('storyId')+'($|[^\d])'))
-                    if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
-                        storylink = None
-                    if storylink:
-                        offset = int(page['href'].split('=')[-1]) # offset is last.
-                        break
-
-            # for reasons I don't understand, searching for story
-            # links by regex wasn't working reliably.  It was missing
-            # the javascript links sometimes.  This is cleaner anyway.
-            for i, div in enumerate(seriessoup.findAll('div', {'class':'title'})):
-                a = div.find('a') # first a is story link.
-                # skip 'report this' and 'TOC' links
-                if a == storylink:
-                    self.setSeries(series_name, 1+i+offset)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-            
-        except Exception, e:
-            print("Series parsing failed: %s"%e)
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -1,244 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DestinysGatewayComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DestinysGatewayComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dgrfa')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.destinysgateway.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dokugacom.py

@@ -1,278 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DokugaComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DokugaComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        
-
-        # www.dokuga.com has two 'sections', shown in URL as
-        # 'fanfiction' and 'spark' that change how things should be
-        # handled.
-        # http://www.dokuga.com/fanfiction/story/7528/1
-        # http://www.dokuga.com/spark/story/7299/1
-        self.section=self.parsedUrl.path.split('/',)[1]
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dkg')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        if 'fanfiction' in self.section:
-            self.dateformat = "%d %b %Y"
-        else:
-            self.dateformat = "%m-%d-%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.dokuga.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfiction/story/1234/1 http://"+cls.getSiteDomain()+"/spark/story/1234/1"
-
-    def getSiteURLPattern(self):
-        return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'The author has disabled anonymous viewing for this story.' in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url,soup):
-        params = {}
-
-        if self.password:
-            params['username'] = self.username
-            params['passwd'] = self.password
-        else:
-            params['username'] = self.getConfig("username")
-            params['passwd'] = self.getConfig("password")
-        params['Submit'] = 'Submit'
-
-        # copy all hidden input tags to pick up appropriate tokens.
-        for tag in soup.findAll('input',{'type':'hidden'}):
-            params[tag['name']] = tag['value']
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['username']))
-    
-        d = self._postUrl(loginUrl, params)
-
-        if "Your session has expired. Please log in again." in d:
-            d = self._postUrl(loginUrl, params)
-
-        if "Logout" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['username']))
-            raise exceptions.FailedToLogin(url,params['username'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-                
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url,soup)
-            data = self._fetchUrl(url)
-            soup = bs.BeautifulSoup(data)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'align' : 'center'}).find('h3')
-        
-        # Find authorid and URL from... author url.
-        aut = a.find('a')
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        alink='http://'+self.host+aut['href']
-        self.story.setMetadata('authorUrl','http://'+self.host+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        a = a.string[:(len(a.string)-4)]
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find the chapters:
-        chapters = soup.find('select').findAll('option')
-        if len(chapters)==1:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1'))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(alink))
-        
-        if 'fanfiction' in self.section:
-            asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
-        
-            #grab the rest of the metadata from the author's page
-            for div in asoup.findAll('div'):
-                nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
-                if nav != None:
-                    break
-            div=div.nextSibling
-            self.setDescription(url,div)
-        
-            div=div.nextSibling
-            
-            a=div.text.split('Rating: ')
-            if len(a) == 2: self.story.setMetadata('rating', a[1].split('&')[0])
-        
-            a=div.text.split('Status: ')
-            if len(a)==2:
-                iscomp=a[1].split('&')[0]
-                if 'Complete' in iscomp:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-            
-            a=div.text.split('Category: ')
-            if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
-        
-            a=div.text.split('Created: ')
-            if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
-        
-            a=div.text.split('Updated: ')
-            if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1]), self.dateformat))
-            
-            div=div.nextSibling.nextSibling
-            a=div.text.split('Words: ')
-            if len(a) == 2: self.story.setMetadata('numWords', a[1].split('&')[0])
-        
-            a=div.text.split('Genre: ')
-            if len(a) == 2:
-                for genre in a[1].split('&')[0].split(', '):
-                    self.story.addToList('genre',genre)
-        
-        else:
-            asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
-            for div in asoup.findAll('div'):
-                nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
-                if nav != None:
-                    break
-                    
-            div=div.nextSibling.nextSibling
-            self.setDescription(url,div)
-            self.story.addToList('category', 'Spark')
-            
-            div=div.nextSibling.nextSibling
-            a=div.text.split('Rating: ')
-            if len(a) == 2: self.story.setMetadata('rating', a[1].split(' - ')[0])
-            
-            a=div.text.split('Status: ')
-            if len(a)==2:
-                iscomp=a[1].split(' - ')[0]
-                if 'Complete' in iscomp:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-            
-            a=div.text.split('Genre: ')
-            if len(a)==2:
-                for genre in a[1].split(' - ')[0].split('/'):
-                    self.story.addToList('genre',genre)
-                
-            div=div.nextSibling.nextSibling
-            
-            a=div.text.split('Updated: ')
-            if len(a)==2:
-                date=a[1].split('      -')[0]
-                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
-        
-                # does not have published date anywhere
-                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-            
-            a=div.text.split('Words ')
-            if len(a)==2: self.story.setMetadata('numWords', a[1])
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'chtext'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -1,217 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DotMoonNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DotMoonNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL. www.dotmoon.net/library_view.php?storyid=3
-        self._setURL('http://' + self.getSiteDomain() + '/library_view.php?storyid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dotm')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y-%m-%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.dotmoon.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/library_view.php?storyid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/library_view.php?storyid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'You must be logged in to read adult-rated stories' in data \
-                or 'Password incorrect' in data \
-                or "That username does not exist" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['user'] = self.username
-            params['passwrd'] = self.password
-        else:
-            params['user'] = self.getConfig("username")
-            params['passwrd'] = self.getConfig("password")
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/board/index.php'
-        
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['user']))
-    
-        d = self._fetchUrl(loginUrl+'?action=login2&user='+params['user']+'&passwrd='+params['passwrd'])
-        d = self._fetchUrl(loginUrl)
-    
-        if "Show unread posts since last visit" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['user']))
-            raise exceptions.FailedToLogin(url,params['user'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Invalid story ID" in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Invalid story ID.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        body=soup.findAll('body')[1]
-        body.find('table').extract()
-        
-        ## Title
-        a = body.find('b')
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
-        a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters: 'library_storyview.php?chapterid=3
-        chapters=body.findAll('a', href=re.compile(r"library_storyview.php\?chapterid=\d+$"))
-        if len(chapters)==0:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.")
-        if len(chapters)==1:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # other tags
-
-        labels = body.find('table', {'width':'390'}).findAll('td')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-            
-            if label != None:
-                if 'Fandom' in label:
-                    self.story.addToList('category',value.string)
-                
-                if 'Setting' in label:
-                    self.story.addToList('genre',value.string)
-                
-                if 'Genre' in label:
-                    self.story.addToList('genre',value.string)
-                
-                if 'Style' in label:
-                    self.story.addToList('genre',value.string)
-                
-                if 'Rating' in label:
-                    self.story.addToList('rating',value.string)
-                
-                if 'Created' in label:
-                    self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-                if 'Updated' in label:
-                    self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-        
-                if 'Status' in label:
-                    if 'Completed' in value.string:
-                        self.story.setMetadata('status', 'Completed')
-                    else:
-                        self.story.setMetadata('status', 'In-Progress')
-                        
-        table=body.findAll('table', {'width':'400'})[1].find('td')
-        self.setDescription(url,stripHTML(table).split('Summary: ')[1])
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('blockquote')
-        div.name='div'
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -1,302 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DracoAndGinnyComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DracoAndGinnyComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dcagn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.dracoandginny.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=2"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        content=soup.find('div',{'class' : 'listbox'})
-        
-        self.setDescription(url,content.find('blockquote'))
-        
-        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
-            self.story.addToList('genre',genre.string)
-
-        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
-            self.story.addToList('warnings',warning.string)
-        
-        labels = content.findAll('b')
-        
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value.split(' |')[0])
-                
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.nextSibling).split(' |')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'class' : 'listbox'})
-        
-        if None == div:
-            div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -1,311 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DramioneOrgAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DramioneOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','drmn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'dramione.org'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Stories that are suitable for ages 16 and older" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Use banner as cover if found
-        coverurl = ''
-        img = soup.find('img',{'class':'banner'})
-        if img:
-            coverurl = img['src']
-            #print "Cover: "+coverurl
-        a = soup.find(text="This story has a banner; click to view.")
-        if a:
-            #print "A: "+ ', '.join("(%s, %s)" %tup for tup in a.parent.attrs)
-            coverurl = a.parent['href']
-            #print "Cover: "+coverurl
-        if coverurl:
-            self.setCoverImage(url,coverurl)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-		
-        genres=soup.findAll('a', {'class' : "tag-1"})
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-
-        warnings=soup.findAll('a', {'class' : "tag-2"})
-        for warning in warnings:
-            self.story.addToList('warnings',warning.string)
-
-        themes=soup.findAll('a', {'class' : "tag-3"})
-        for theme in themes:
-            self.story.addToList('themes',theme.string)
-
-        hermiones=soup.findAll('a', {'class' : "tag-4"})
-        for hermione in hermiones:
-            self.story.addToList('hermiones',hermione.string)
-
-        dracos=soup.findAll('a', {'class' : "tag-5"})
-        for draco in dracos:
-            self.story.addToList('dracos',draco.string)
-
-        timelines=soup.findAll('a', {'class' : "tag-6"})
-        for timeline in timelines:
-            self.story.addToList('timeline',timeline.string)
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Read' in label:
-                self.story.setMetadata('read', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-        try:
-            self.story.setMetadata('reviews',
-                                   stripHTML(soup.find('h2',{'id':'pagetitle'}).
-                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -1,224 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return EfictionEstelielDeAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class EfictionEstelielDeAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','eesd')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'efiction.esteliel.de'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-       
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        pagetitle = soup.find('div',{'id':'pagetitle'})
-        ## Title
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        list = soup.find('div', {'class':'listbox'})
-        labelspan=list.find('span',{'class':'label'})
-        value = labelspan.nextSibling
-        label = labelspan.string
-        genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-        
-        labels = list.findAll('b')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while 'Rating' not in str(value):
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Words' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Category' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            if list.find('a', href=re.compile(r"series.php")) != None:
-                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
-                    # Find Series name from series URL.
-                    series_url = 'http://'+self.host+'/'+series['href']
-                    # use BeautifulSoup HTML parser to make everything easier to find.
-                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-                    i=1
-                    for a in storyas:
-                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                            name=seriessoup.find('div', {'id' : 'pagetitle'})
-                            name.find('a').extract()
-                            self.setSeries(name.text.split(' by[')[0], i)
-                            self.story.setMetadata('seriesUrl',series_url)
-                            i=0
-                            break
-                        i+=1
-                    if i == 0:
-                        break
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-                                     
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -1,316 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return EFPFanFicNet
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class EFPFanFicNet(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','efp')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.efpfanfic.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if( 'Fai il login e leggi la storia!' in data or
-            'Questa storia presenta contenuti non adatti ai minori' in data ):
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Invia'
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?sid='+self.story.getMetadata('storyId')
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if '<a class="menu" href="newaccount.php">' in d : # register for new account link
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'sid' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
-            for o in allOptions:
-                url = u'http://%s/%s' % ( self.getSiteDomain(),
-                                          o['value'])
-                # just in case there's tags, like <i> in chapter titles.
-                title = stripHTML(o)
-                self.chapterUrls.append((title,url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        self.story.setMetadata('language','Italian')
-        
-        # normalize story URL to first chapter if later chapter URL was given:
-        url = self.chapterUrls[0][1].replace('&i=1','')
-        logger.debug("Normalizing to URL: "+url)
-        self._setURL(url)
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        storya = None
-        authsoup = None
-        storyblock = None
-        authurl = self.story.getMetadata('authorUrl')
-        
-        ## author can have more than one page of stories.
-        while storyblock == None:
-
-            # no storya, but do have authsoup--we're looping on author pages.
-            if authsoup != None:
-                # last author link with offset should be the 'next' link.
-                authurl = u'http://%s/%s' % ( self.getSiteDomain(),
-                                              authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
-            
-            # Need author page for most of the metadata.
-            logger.debug("fetching author page: (%s)"%authurl)
-            authsoup = bs.BeautifulSoup(self._fetchUrl(authurl))
-            #print("authsoup:%s"%authsoup)
-
-            storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
-            for storya in storyas:
-                #print("======storya:%s"%storya)
-                storyblock = storya.findParent('div',{'class':'storybloc'})
-                #print("======storyblock:%s"%storyblock)
-                if storyblock != None:
-                    continue
-
-        self.setDescription(url,storyblock.find('div', {'class':'introbloc'}))
-
-        noteblock = storyblock.find('div', {'class':'notebloc'})
-        #print("%s"%noteblock)
-        notetext = ("%s" % noteblock).replace("<br />"," |")
-        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />
-        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />
-        # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>
-
-        cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-            
-        for item in notetext.split("|"):
-            if ":" in item:
-                (label,value) = item.split(":")
-                label=label.strip()
-                value=value.strip()
-            else:
-                label=value=item.strip()
-
-            if 'Pubblicata' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Aggiornata' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-            if label == "Completa":
-                self.story.setMetadata('status', 'Completed')
-            
-            if label == "In corso":
-                self.story.setMetadata('status', 'In-Progress')
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Personaggi' in label:
-                for val in value.split(","):
-                    self.story.addToList('characters',val)
-                    
-            if 'Genere' in label:
-                for val in value.split(","):
-                    self.story.addToList('genre',val)
-                    
-            if 'Coppie' in label:
-                for val in value.split(","):
-                    self.story.addToList('ships',val)
-
-            if 'Avvertimenti' in label:
-                for val in value.split(","):
-                    if val != "None":
-                        self.story.addToList('warnings',val)
-
-            # 'extra' metadata for this adapter:
-                    
-            if 'Tipo di coppia' in label:
-                for val in value.split(","):
-                    self.story.addToList('type',val)
-                    
-            if 'Note' in label:
-                for val in value.split(","):
-                    if val != "None":
-                        self.story.addToList('notes',val)
-                    
-            if 'Contesto' in label:
-                self.story.setMetadata('context', value)
-
-            ## Note--efp doesn't provide word count.
-                
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?ssid=\d+&i=1"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
-        
-        div = soup.find('div', {'class' : 'storia'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        # remove any header and 'o:p' tags.
-        for tag in div.findAll("head") + div.findAll("o:p"):
-            tag.extract()
-
-        # change any html and body tags to div.
-        for tag in div.findAll("html") + div.findAll("body"):
-            tag.name='div'
-
-        # remove extra bogus doctype.
-        #<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
-        return re.sub(r"<!DOCTYPE[^>]+>","",self.utf8FromSoup(url,div))

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -1,256 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ErosnSapphoSycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','essph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'erosnsappho.sycophanthex.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=18"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        rating=pt.text.split('(')[1].split(')')[0]
-        self.story.setMetadata('rating', rating)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-				
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-		
-        labels = soup.findAll('span',{'class':'label'})
-		
-        value = labels[0].previousSibling
-        svalue = ""
-        while value != None:
-            val = value
-            value = value.previousSibling
-        while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-		
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value.split(' -')[0])
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Complete' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanficcastletvnet.py

@@ -1,322 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2014 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# In general an 'adapter' needs to do these five things:
-
-# - 'Register' correctly with the downloader
-# - Site Login (if needed)
-# - 'Are you adult?' check (if needed--some do one, some the other, some both)
-# - Grab the chapter list
-# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
-# - Grab the chapter texts
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return FanficCastleTVNetAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','csltv') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfic.castletv.net' # XXX
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-         ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=3"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        pagetitle = soup.find('div',{'id':'pagetitle'})
-        ## Title
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        
-        # Reviews
-        reviewdata = soup.find('div', {'id' : 'sort'})
-        a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
-        self.story.setMetadata('reviews',stripHTML(a)) 
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfichu.py

@@ -1,185 +0,0 @@
-# coding=utf-8
-
-import re
-import urllib2
-import urlparse
-
-from .. import BeautifulSoup
-
-from base_adapter import BaseSiteAdapter, makeDate
-from .. import exceptions
-
-
-_SOURCE_CODE_ENCODING = 'utf-8'
-
-
-def getClass():
-    return FanficHuAdapter
-
-
-def _get_query_data(url):
-    components = urlparse.urlparse(url)
-    query_data = urlparse.parse_qs(components.query)
-    return dict((key, data[0]) for key, data in query_data.items())
-
-
-class FanficHuAdapter(BaseSiteAdapter):
-    SITE_ABBREVIATION = 'ffh'
-    SITE_DOMAIN = 'fanfic.hu'
-    SITE_LANGUAGE = 'Hungarian'
-
-    BASE_URL = 'http://' + SITE_DOMAIN + '/merengo/'
-    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%s'
-
-    DATE_FORMAT = '%m/%d/%Y'
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        query_data = urlparse.parse_qs(self.parsedUrl.query)
-        story_id = query_data['sid'][0]
-
-        self.story.setMetadata('storyId', story_id)
-        self._setURL(self.VIEW_STORY_URL_TEMPLATE % story_id)
-        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
-        self.story.setMetadata('language', self.SITE_LANGUAGE)
-
-    def _customized_fetch_url(self, url, exception=None, parameters=None):
-        if exception:
-            try:
-                data = self._fetchUrl(url, parameters)
-            except urllib2.HTTPError:
-                raise exception(self.url)
-        # Just let self._fetchUrl throw the exception, don't catch and
-        # customize it.
-        else:
-            data = self._fetchUrl(url, parameters)
-
-        return BeautifulSoup.BeautifulSoup(data)
-
-    @staticmethod
-    def getSiteDomain():
-        return FanficHuAdapter.SITE_DOMAIN
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return cls.VIEW_STORY_URL_TEMPLATE % 1234
-
-    def getSiteURLPattern(self):
-        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
-
-    def extractChapterUrlsAndMetadata(self):
-        soup = self._customized_fetch_url(self.url + '&i=1')
-
-        if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
-            raise exceptions.StoryDoesNotExist(self.url)
-
-        chapter_options = soup.find('form', action='viewstory.php').select('option')
-        # Remove redundant "Fejezetek" option
-        chapter_options.pop(0)
-
-        # If there is still more than one entry remove chapter overview entry
-        if len(chapter_options) > 1:
-            chapter_options.pop(0)
-
-        for option in chapter_options:
-            url = urlparse.urljoin(self.url, option['value'])
-            self.chapterUrls.append((option.string, url))
-
-        author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
-        soup = self._customized_fetch_url(author_url)
-
-        story_id = self.story.getMetadata('storyId')
-        for table in soup('table', {'class': 'mainnav'}):
-            title_anchor = table.find('span', {'class': 'storytitle'}).a
-            href = title_anchor['href']
-            if href.startswith('javascript:'):
-                href = href.rsplit(' ', 1)[1].strip("'")
-            query_data = _get_query_data(href)
-
-            if query_data['sid'] == story_id:
-                break
-        else:
-            # This should never happen, the story must be found on the author's
-            # page.
-            raise exceptions.FailedToDownload(self.url)
-
-        self.story.setMetadata('title', title_anchor.string)
-
-        rows = table('tr')
-
-        anchors = rows[0].div('a')
-        author_anchor = anchors[1]
-        query_data = _get_query_data(author_anchor['href'])
-        self.story.setMetadata('author', author_anchor.string)
-        self.story.setMetadata('authorId', query_data['uid'])
-        self.story.setMetadata('authorUrl', urlparse.urljoin(self.BASE_URL, author_anchor['href']))
-        self.story.setMetadata('reviews', anchors[3].string)
-
-        if self.getConfig('keep_summary_html'):
-            self.story.setMetadata('description', self.utf8FromSoup(author_url, rows[1].td))
-        else:
-            self.story.setMetadata('description', ''.join(rows[1].td(text=True)))
-
-        for row in rows[3:]:
-            index = 0
-            cells = row('td')
-
-            while index < len(cells):
-                cell = cells[index]
-                key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
-                try:
-                    value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
-                except AttributeError:
-                    value = None
-
-                if key == 'Kategória':
-                    for anchor in cells[index+1]('a'):
-                        self.story.addToList('category', anchor.string)
-
-                elif key == 'Szereplõk':
-                    if cells[index+1].string:
-                        for name in cells[index+1].string.split(', '):
-                            self.story.addToList('character', name)
-
-                elif key == 'Korhatár':
-                    if value != 'nem korhatáros':
-                        self.story.setMetadata('rating', value)
-
-                elif key == 'Figyelmeztetések':
-                    for b_tag in cells[index+1]('b'):
-                        self.story.addToList('warnings', b_tag.string)
-
-                elif key == 'Jellemzõk':
-                    for genre in cells[index+1].string.split(', '):
-                        self.story.addToList('genre', genre)
-
-                elif key == 'Fejezetek':
-                    self.story.setMetadata('numChapters', int(value))
-
-                elif key == 'Megjelenés':
-                    self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
-
-                elif key == 'Frissítés':
-                    self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
-
-                elif key == 'Szavak':
-                    self.story.setMetadata('numWords', value)
-
-                elif key == 'Befejezett':
-                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
-
-                index += 2
-
-        if self.story.getMetadata('rating') == '18':
-            if not (self.is_adult or self.getConfig('is_adult')):
-                raise exceptions.AdultCheckRequired(self.url)
-
-    def getChapterText(self, url):
-        soup = self._customized_fetch_url(url)
-        story_cell = soup.find('form', action='viewstory.php').parent.parent
-
-        for div in story_cell('div'):
-            div.extract()
-
-        return self.utf8FromSoup(url, story_cell)

fanficdownloader/adapters/adapter_fanfictioncsodaidokhu.py

@@ -1,218 +0,0 @@
-# coding=utf-8
-
-import re
-import urllib2
-import urlparse
-
-from .. import BeautifulSoup
-
-from base_adapter import BaseSiteAdapter, makeDate
-from .. import exceptions
-
-
-_SOURCE_CODE_ENCODING = 'utf-8'
-
-
-def getClass():
-    return FanfictionCsodaidokHuAdapter
-
-
-def _get_query_data(url):
-    components = urlparse.urlparse(url)
-    query_data = urlparse.parse_qs(components.query)
-    return dict((key, data[0]) for key, data in query_data.items())
-
-
-# yields Tag _and_ NavigableString siblings from the given tag. The
-# BeautifulSoup findNextSiblings() method for some reasons only returns either
-# NavigableStrings _or_ Tag objects, not both.
-def _yield_next_siblings(tag):
-    sibling = tag.nextSibling
-    while sibling:
-        yield sibling
-        sibling = sibling.nextSibling
-
-
-class FanfictionCsodaidokHuAdapter(BaseSiteAdapter):
-    _SITE_DOMAIN = 'fanfiction.csodaidok.hu'
-    _BASE_URL = 'http://' + _SITE_DOMAIN + '/'
-    _VIEW_STORY_URL_TEMPLATE = _BASE_URL + 'viewstory.php?sid=%s'
-    _VIEW_CHAPTER_URL_TEMPLATE = _VIEW_STORY_URL_TEMPLATE + '&chapter=%s'
-
-    _STORY_DOES_NOT_EXIST_PAGE_TITLE = 'Cím:  Szerző:'
-    _DATE_FORMAT = '%Y.%m.%d'
-    _SITE_LANGUAGE = 'Hungarian'
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        query_data = urlparse.parse_qs(self.parsedUrl.query)
-        story_id = query_data['sid'][0]
-
-        self.story.setMetadata('storyId', story_id)
-        self._setURL(self._VIEW_STORY_URL_TEMPLATE % story_id)
-        self.story.setMetadata('siteabbrev', self._SITE_DOMAIN)
-        self.story.setMetadata('language', self._SITE_LANGUAGE)
-
-    def _customized_fetch_url(self, url, exception=None, parameters=None):
-        if exception:
-            try:
-                data = self._fetchUrl(url, parameters)
-            except urllib2.HTTPError:
-                raise exception(self.url)
-        # Just let self._fetchUrl throw the exception, don't catch and
-        # customize it.
-        else:
-            data = self._fetchUrl(url, parameters)
-
-        return BeautifulSoup.BeautifulSoup(data)
-
-    @staticmethod
-    def getSiteDomain():
-        return FanfictionCsodaidokHuAdapter._SITE_DOMAIN
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return cls._VIEW_STORY_URL_TEMPLATE % 1234
-
-    def getSiteURLPattern(self):
-        return re.escape(self._VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
-
-    def extractChapterUrlsAndMetadata(self):
-        soup = self._customized_fetch_url(self.url + '&chapter=1')
-
-        element = soup.find('div', id='pagetitle')
-        page_title = ''.join(element(text=True)).encode(_SOURCE_CODE_ENCODING)
-        if page_title == self._STORY_DOES_NOT_EXIST_PAGE_TITLE:
-            raise exceptions.StoryDoesNotExist(self.url)
-
-        author_url = urlparse.urljoin(self.url, element.a['href'])
-
-        story_id = self.story.getMetadata('storyId')
-        element = soup.find('select', {'name': 'chapter'})
-        if element:
-            for option in element('option'):
-                title = option.string
-                url = self._VIEW_CHAPTER_URL_TEMPLATE % (story_id, option['value'])
-                self.chapterUrls.append((title, url))
-
-        soup = self._customized_fetch_url(author_url)
-        story_id = self.story.getMetadata('storyId')
-
-        for listbox_div in soup('div', {'class': lambda klass: klass and 'listbox' in klass}):
-            a = listbox_div.div.a
-            if not a['href'].startswith('viewstory.php?sid='):
-                continue
-
-            query_data = _get_query_data(a['href'])
-            if query_data['sid'] == story_id:
-                break
-        else:
-            raise exceptions.FailedToDownload(self.url)
-
-        title = ''.join(a(text=True))
-        self.story.setMetadata('title', title)
-        if not self.chapterUrls:
-            self.chapterUrls.append((title, self.url))
-
-        element = a.findNextSibling('a')
-        self.story.setMetadata('author', element.string)
-        query_data = _get_query_data(element['href'])
-        self.story.setMetadata('authorId', query_data['uid'])
-        self.story.setMetadata('authorUrl', author_url)
-
-        element = element.findNextSibling('span')
-        rating = element.nextSibling.strip(' [')
-
-        if rating.encode(_SOURCE_CODE_ENCODING) != 'Korhatár nélkül':
-            self.story.setMetadata('rating', rating)
-
-        if rating == '18':
-            raise exceptions.AdultCheckRequired(self.url)
-
-        element = element.findNextSiblings('a')[1]
-        self.story.setMetadata('reviews', element.string)
-
-        sections = listbox_div('div', {'class': lambda klass: klass and klass in ['content', 'tail']})
-        for section in sections:
-            for element in section('span', {'class': 'classification'}):
-                key = element.string.encode(_SOURCE_CODE_ENCODING).strip(' :')
-                try:
-                    value = element.nextSibling.string.encode(_SOURCE_CODE_ENCODING).strip()
-                except AttributeError:
-                    value = None
-
-                if key == 'Tartalom':
-                    contents = []
-                    keep_summary_html = self.getConfig('keep_summary_html')
-
-                    for sibling in _yield_next_siblings(element):
-                        if isinstance(sibling, BeautifulSoup.Tag):
-                            if sibling.name == 'span' and sibling.get('class', None) == 'classification':
-                                break
-
-                            if keep_summary_html:
-                                contents.append(self.utf8FromSoup(author_url, sibling))
-                            else:
-                                contents.append(''.join(sibling(text=True)))
-                        else:
-                            contents.append(sibling)
-                    self.story.setMetadata('description', ''.join(contents))
-
-                elif key == 'Kategória':
-                    for sibling in element.findNextSiblings(['a', 'span']):
-                        if sibling.name == 'span':
-                            break
-
-                        self.story.addToList('category', sibling.string)
-
-                elif key == 'Szereplők':
-                    for name in value.split(', '):
-                        self.story.addToList('characters', name)
-
-                elif key == 'Műfaj':
-                    if value != 'Nincs':
-                        self.story.setMetadata('genre', value)
-
-                elif key == 'Figyelmeztetés':
-                    if value != 'Nincs':
-                        for warning in value.split(', '):
-                            self.story.addToList('warnings', warning)
-
-                elif key == 'Kihívás':
-                    if value != 'Nincs':
-                        self.story.setMetadata('challenge', value)
-
-                elif key == 'Sorozat':
-                    if value != 'Nincs':
-                        self.story.setMetadata('series', value)
-
-                elif key == 'Fejezetek':
-                    self.story.setMetadata('numChapters', int(value))
-
-                elif key == 'Befejezett':
-                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
-
-                elif key == 'Szavak száma':
-                    self.story.setMetadata('numWords', value)
-
-                elif key == 'Feltöltve':
-                    self.story.setMetadata('datePublished', makeDate(value, self._DATE_FORMAT))
-
-                elif key == 'Frissítve':
-                    self.story.setMetadata('dateUpdated', makeDate(value, self._DATE_FORMAT))
-
-    def getChapterText(self, url):
-        soup = self._customized_fetch_url(url)
-        contents = []
-
-        notes_div = soup.find('div', id='notes')
-        if notes_div:
-            contents.append(self.utf8FromSoup(url, notes_div))
-            story_div = notes_div.findNextSibling('div')
-        else:
-            element = soup.find('div', {'class': 'jumpmenu'})
-            story_div = element.findNextSibling('div')
-
-        contents.append(self.utf8FromSoup(url, story_div.span))
-        return ''.join(contents)

fanficdownloader/adapters/adapter_fanfictionjunkiesde.py

@@ -1,291 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# By virtue of being recent and requiring both is_adult and user/pass,
-# adapter_fanficcastletvnet.py is the best choice for learning to
-# write adapters--especially for sites that use the eFiction system.
-# Most sites that have ".../viewstory.php?sid=123" in the story URL
-# are eFiction.
-
-# For non-eFiction sites, it can be considerably more complex, but
-# this is still a good starting point.
-
-# In general an 'adapter' needs to do these five things:
-
-# - 'Register' correctly with the downloader
-# - Site Login (if needed)
-# - 'Are you adult?' check (if needed--some do one, some the other, some both)
-# - Grab the chapter list
-# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
-# - Grab the chapter texts
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return FanfictionJunkiesDeAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/efiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ffjde') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction-junkies.de' # XXX
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/efiction/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=1" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "For adults only " in data: # XXX 
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        pagetitle = soup.find('h4')
-        ## Title      
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/efiction/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        
-        # Reviews
-        reviewdata = soup.find('div', {'id' : 'sort'})
-        a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
-        self.story.setMetadata('reviews',stripHTML(a)) 
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                           
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        list = soup.find('div', {'class':'listbox'})
-        
-        
-        labels = list.findAll('b')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Zusammenfassung' in label:
-                self.setDescription(url,value)
-
-            if 'Eingestuft' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'W&ouml;rter' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Kategorie' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Charaktere' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Abgeschlossen' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Ver&ouml;ffentlicht' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Aktualisiert' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/efiction/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -16,24 +16,16 @@
 #
 
 import time
-from datetime import datetime
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
-from urllib import unquote_plus
 import time
 
 from .. import BeautifulSoup as bs
 from .. import exceptions as exceptions
-from ..htmlcleanup import stripHTML
 
 from base_adapter import BaseSiteAdapter,  makeDate
 
-ffnetgenres=["Adventure", "Angst", "Crime", "Drama", "Family", "Fantasy", "Friendship", "General",
-             "Horror", "Humor", "Hurt-Comfort", "Mystery", "Parody", "Poetry", "Romance", "Sci-Fi",
-             "Spiritual", "Supernatural", "Suspense", "Tragedy", "Western"]
-
 class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -44,7 +36,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
 
         # normalized story URL.
-        self._setURL("https://"+self.getSiteDomain()\
+        self._setURL("http://"+self.getSiteDomain()\
                          +"/s/"+self.story.getMetadata('storyId')+"/1/")
 
         # ffnet update emails have the latest chapter URL.
@@ -52,14 +44,10 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # latest chapter yet and going back to chapter 1 to pull the
         # chapter list doesn't get the latest.  So save and use the
         # original URL given to pull chapter list & metadata.
-        # Not used by plugin because URL gets normalized first for
-        # eliminating duplicate story urls.
         self.origurl = url
-        if "https://m." in self.origurl:
+        if "http://m." in self.origurl:
             ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("https://m.","https://www.")
-
-        self.opener.addheaders.append(('Referer',self.origurl))
+            self.origurl = self.origurl.replace("http://m.","http://www.")
 
     @staticmethod
     def getSiteDomain():
@@ -69,41 +57,24 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.fanfiction.net','m.fanfiction.net']
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://www.fanfiction.net/s/1234/1/ https://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
+    def getSiteExampleURLs(self):
+        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
 
     def getSiteURLPattern(self):
-        return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
+        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
 
-    def _fetchUrl(self,url,parameters=None,extrasleep=1.0,usecache=True):
-        ## ffnet(and, I assume, fpcom) tends to fail more if hit too
-        ## fast.  This is in additional to what ever the
-        ## slow_down_sleep_time setting is.
-        return BaseSiteAdapter._fetchUrl(self,url,
-                                         parameters=parameters,
-                                         extrasleep=extrasleep,
-                                         usecache=usecache)
-
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
-    
-    def doExtractChapterUrlsAndMetadata(self,get_cover=True):
+    def extractChapterUrlsAndMetadata(self):
 
         # fetch the chapter.  From that we will get almost all the
         # metadata and chapter list
 
         url = self.origurl
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
-            #logger.debug("\n===================\n%s\n===================\n"%data)
+            #print("\n===================\n%s\n===================\n"%data)
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -111,171 +82,99 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             else:
                 raise e
             
-        if "Unable to locate story" in data:
+        if "Unable to locate story with id of " in data:
             raise exceptions.StoryDoesNotExist(url)
 
         # some times "Chapter not found...", sometimes "Chapter text not found..."
         if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
-        if self.getConfig('check_next_chapter'):
+        try:
+            # rather nasty way to check for a newer chapter.  ffnet has a
+            # tendency to send out update notices in email before all
+            # their servers are showing the update on the first chapter.
             try:
-                ## ffnet used to have a tendency to send out update
-                ## notices in email before all their servers were
-                ## showing the update on the first chapter.  It
-                ## generates another server request and doesn't seem
-                ## to be needed lately, so now default it to off.
-                try:
-                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-                # get chapter part of url.
-                except:
-                    chapcount = 1
-                chapter = url.split('/',)[5]
-                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                                self.story.getMetadata('storyId'),
-                                                chapcount+1)
-                logger.debug('=Trying newer chapter: %s' % tryurl)
-                newdata = self._fetchUrl(tryurl)
-                if "not found. Please check to see you are not using an outdated url." \
-                        not in newdata:
-                    logger.debug('=======Found newer chapter: %s' % tryurl)
-                    soup = bs.BeautifulSoup(newdata)
+                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+            # get chapter part of url.
             except:
-                pass
-            
+                chapcount = 1
+            chapter = url.split('/',)[5]
+            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                           self.story.getMetadata('storyId'),
+                                           chapcount+1)
+            print('=Trying newer chapter: %s' % tryurl)
+            newdata = self._fetchUrl(tryurl)
+            if "not found. Please check to see you are not using an outdated url." \
+                    not in newdata:
+                print('=======Found newer chapter: %s' % tryurl)
+                soup = bs.BeautifulSoup(newdata)
+        except:
+            pass
+        
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','https://'+self.host+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
 
-        ## Pull some additional data from html.
-
-        ## ffnet shows category two ways
-        ## 1) class(Book, TV, Game,etc) >> category(Harry Potter, Sailor Moon, etc)
-        ## 2) cat1_cat2_Crossover
-        ## For 1, use the second link.
-        ## For 2, fetch the crossover page and pull the two categories from there.
-
-        categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
-        #print("xcontrast_txt a:%s"%categories)
-        if len(categories) > 1:
-            # Strangely, the ones with *two* links are the
-            # non-crossover categories.  Each is in a category itself
-            # of Book, Movie, etc.
-            self.story.addToList('category',stripHTML(categories[1]))
-        elif 'Crossover' in categories[0]['href']:
-            caturl = "https://%s%s"%(self.getSiteDomain(),categories[0]['href'])
-            catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
-            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/.+?/\d+/")):
-                self.story.addToList('category',stripHTML(a))
-            else:
-                # Fall back.  I ran across a story with a Crossver
-                # category link to a broken page once.
-                # http://www.fanfiction.net/s/2622060/1/
-                # Naruto + Harry Potter Crossover
-                logger.info("Fall back category collection")
-                for c in stripHTML(categories[0]).replace(" Crossover","").split(' + '):
-                    self.story.addToList('category',c)
-                
-                
             
-        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
-        rating = a.string
-        if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
-            rating = rating[8:]
+        # start by finding a script towards the bottom that has a
+        # bunch of useful stuff in it.
             
-        self.story.setMetadata('rating',rating)
+        # var storyid = 6577076;
+        # var chapter = 1;
+        # var chapters = 17;
+        # var words = 42787;
+        # var userid = 2645830;
+        # var title = 'The+Invitation';
+        # var title_t = 'The Invitation';
+        # var summary = 'Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter\'s wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story.';
+        # var categoryid = 224;
+        # var cat_title = 'Harry Potter';
+        # var datep = '12-21-10';
+        # var dateu = '04-06-11';
+        # var author = 'U n F a b u l o u s M e';
 
-        # after Rating, the same bit of text containing id:123456 contains
-        # Complete--if completed.
-        gui_table1i = soup.find('div',{'id':'content_wrapper_inner'})
-
-        self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
-
-        summarydiv = gui_table1i.find('div',{'style':'margin-top:2px'})
-        if summarydiv:
-            self.setDescription(url,stripHTML(summarydiv))
-            
-
-        grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
-        # for b in grayspan.findAll('button'):
-        #     b.extract()
-        metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
-        #logger.debug("metatext:(%s)"%metatext)
-        metalist = metatext.split(" - ")
-        #logger.debug("metalist:(%s)"%metalist)
-
-        # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
-        # Rated: Fiction T - English - Adventure/Sci-Fi - Naruto U. - Chapters: 22 - Words: 114,414 - Reviews: 395 - Favs: 779 - Follows: 835 - Updated: 03-21-13 - Published: 04-28-12 - id: 8067258 
-
-        # rating is obtained above more robustly.
-        if metalist[0].startswith('Rated:'):
-            metalist=metalist[1:]
-
-        # next is assumed to be language.
-        self.story.setMetadata('language',metalist[0])
-        metalist=metalist[1:]
-
-        # next might be genre.
-        genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
-        goodgenres=True
-        for g in genrelist:
-            #logger.debug("g:(%s)"%g)
-            if g.strip() not in ffnetgenres:
-                #logger.info("g not in ffnetgenres")
-                goodgenres=False
-        if goodgenres:
-            self.story.extendList('genre',genrelist)
-            metalist=metalist[1:]
-
-        # Updated: <span data-xutime='1368059198'>5/8</span> - Published: <span data-xutime='1278984264'>7/12/2010</span>
-        # Published: <span data-xutime='1384358726'>8m ago</span>
-        dates = soup.findAll('span',{'data-xutime':re.compile(r'^\d+$')})
-        if len(dates) > 1 :
-            # updated get set to the same as published upstream if not found.
-            self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
-        self.story.setMetadata('datePublished',datetime.fromtimestamp(float(dates[-1]['data-xutime'])))
-            
-        donechars = False
-        while len(metalist) > 0:
-            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Updated:') or metalist[0].startswith('Published:'):
-                pass
-            elif  metalist[0].startswith('Reviews'):
-                self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Favs:'):
-                self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Follows:'):
-                self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Words'):
-                self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
-            elif not donechars:
-                # with 'pairing' support, pairings are bracketed w/o comma after
-                # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
-                self.story.extendList('characters',metalist[0].replace('[','').replace(']',',').split(','))
-
-                l = metalist[0]
-                while '[' in l:
-                    self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
-                    l = l[l.index(']')+1:]
-
-                donechars = True
-            metalist=metalist[1:]
-
-        if 'Status: Complete' in metatext:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        if get_cover:
-            # Try the larger image first.
-            try:
-                img = soup.find('img',{'class':'lazy cimage'})
-                self.setCoverImage(url,img['data-original'])
-            except:
-                img = soup.find('img',{'class':'cimage'})
-                if img:
-                    self.setCoverImage(url,img['src'])
+        for script in soup.findAll('script', src=None):
+            if not script:
+                continue
+            if not script.string:
+                continue
+            if 'var storyid' in script.string:
+                for line in script.string.split('\n'):
+                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;\r?$",line)
+                    if m == None : continue
+                    var,value = m.groups()
+                    # remove javascript escaping from values.
+                    value = re.sub(r'\\(.)',r'\1',value)
+                    #print var,value
+                    if 'words' in var:
+                        self.story.setMetadata('numWords', value)
+                    if 'title_t' in var:
+                        self.story.setMetadata('title', value)
+                    if 'summary' in var:
+                        self.setDescription(url,value)
+                        #self.story.setMetadata('description', value)
+                    if 'datep' in var:
+                        self.story.setMetadata('datePublished',makeDate(value, '%m-%d-%y'))
+                    if 'dateu' in var:
+                        self.story.setMetadata('dateUpdated',makeDate(value, '%m-%d-%y'))
+                    if 'cat_title' in var:
+                        if "Crossover" in value:
+                            value = re.sub(r' Crossover$','',value)
+                            for c in value.split(' and '):
+                                self.story.addToList('category',c)
+                                # Screws up when the category itself
+                                # contains ' and '.  But that's rare
+                                # and the only alternative is to find
+                                # the 'Crossover' category URL and
+                                # parse that page to search for <a>
+                                # with href /crossovers/(name)/(num)/
+				# <a href="/crossovers/Harry_Potter/224/">Harry Potter</a>
+				# <a href="/crossovers/Naruto/1402/">Naruto</a>
+                        else:
+                            self.story.addToList('category',value)
+                break # for script in soup.findAll('script', src=None):
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapter' } )
@@ -286,9 +185,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             allOptions = select.findAll('option')
             for o in allOptions:
-                url = u'https://%s/s/%s/%s/' % ( self.getSiteDomain(),
-                                                 self.story.getMetadata('storyId'),
-                                                 o['value'])
+                url = u'http://%s/s/%s/%s/' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('storyId'),
+                                            o['value'])
                 # just in case there's tags, like <i> in chapter titles.
                 title = u"%s" % o
                 title = re.sub(r'<[^>]+>','',title)
@@ -296,41 +195,80 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
+        ## Pull some additional data from html.  Find Rating and look around it.
+
+        a = soup.find('a', href='http://www.fictionratings.com/')
+        self.story.setMetadata('rating',a.string)
+
+        # used below to get correct characters.
+        metatext = a.findNext(text=re.compile(r' - Reviews:'))
+        if metatext == None: # indicates there's no Reviews, look for id: instead.
+            metatext = a.findNext(text=re.compile(r' - id:'))
+
+        # after Rating, the same bit of text containing id:123456 contains
+        # Complete--if completed.
+        if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        # Parse genre(s) from <meta name="description" content="..."
+        # <meta name="description" content="A Transformers/Beast Wars  - Humor fanfiction with characters Prowl & Sideswipe. Story summary: Sideswipe is bored. Prowl appears to be so, too  or at least, Sideswipe thinks he looks bored . So Sideswipe entertains them. After all, what's more fun than a race? Song-fic.">
+        # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
+        # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
+        # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
+        # <meta name="description" content="Chapter 1 of a Harry Potter and Transformers  - Humor/Adventure crossover fanfiction  with characters: Harry P. & Ironhide. IT’s one thing to be tossed thru the Veil for something he didn’t do. It was quite another to wake in his animigus form in a world not his own. Harry just knew someone was laughing at him somewhere. Mech/Mech pairings inside..">
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
+                     soup.find('meta',{'name':'description'})['content'])
+        if m != None:
+            genres=m.group('genres')
+            if genres != None:
+                # Hurt/Comfort is one genre.
+                genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
+                for g in genres.split('/'):
+                    self.story.addToList('genre',g)
+
+            if m.group('chars') != None:
+
+                # At this point we've proven that there's character(s)
+                # We can't reliably parse characters out of meta name="description".
+                # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
+                # But we can pull them from the reviewstext line, now that we know about existance of chars.
+                # reviewstext can take form of:
+                # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
+                # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
+                # - English - Humor/Adventure -  Harry P. & Ironhide - Reviews:
+                mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
+                              metatext)
+                chars = mc.group("chars")
+                for c in chars.split(' & '):
+                    self.story.addToList('characters',c)
+        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
+        if m.group('lang') != None:
+            self.story.setMetadata('language',m.group('lang'))
+                    
         return
 
     def getChapterText(self, url):
-        # time.sleep(4.0) ## ffnet(and, I assume, fpcom) tends to fail
-        #                 ## more if hit too fast.  This is in
-        #                 ## additional to what ever the
-        #                 ## slow_down_sleep_time setting is.
-        logger.debug('Getting chapter text from: %s' % url)
-        data = self._fetchUrl(url,extrasleep=4.0)
-
-        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
-        
-        # some ancient stories have body tags inside them that cause
-        # soup parsing to discard the content.  For story text we
-        # don't care about anything before "<div role='main'" and
-        # this kills any body tags.
-        divstr = "<div role='main'"
-        if divstr not in data:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-        else:
-            data = data[data.index(divstr):] 
-        data = data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
-        
+        logging.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+                        ## more if hit too fast.  This is in
+                        ## additional to what ever the
+                        ## slow_down_sleep_time setting is.
+        data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data)
 
         ## Remove the 'share' button.
         sharediv = soup.find('div', {'class' : 'a2a_kit a2a_default_style'})
         if sharediv:
             sharediv.extract()
+        else:
+            logging.debug('share button div not found')
         
-        div = soup.find('div', {'id' : 'storytextp'})
+        div = soup.find('div', {'id' : 'storytext'})
         
         if None == div:
-            logger.debug('div id=storytextp not found.  data:%s'%data)
+            logging.debug('div id=storytext not found.  data:%s'%data)
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -1,212 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return FanFiktionDeAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class FanFiktionDeAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ffde')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d.%m.%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.fanfiktion.de'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?"
-        
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
-    
-        ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Diese Geschichte wurde als entwicklungsbeeintr' in data \
-                or 'There is no such account on our website' in data \
-                or "Noch kein registrierter Benutzer?" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self,url):
-        params = {}
-
-        if self.password:
-            params['nickname'] = self.username
-            params['passwd'] = self.password
-        else:
-            params['nickname'] = self.getConfig("username")
-            params['passwd'] = self.getConfig("password")
-        params['savelogindata'] = '1'
-        params['a'] = 'l'
-        params['submit'] = 'Login...'
-
-        loginUrl = 'https://ssl.fanfiktion.de/'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['nickname']))
-        soup = bs.BeautifulSoup(self._postUrl(loginUrl,params))
-        if not soup.find('a', title='Logout'):
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['nickname']))
-            raise exceptions.FailedToLogin(url,params['nickname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-                
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url,usecache=False)
-            
-        if "Uhr ist diese Geschichte nur nach einer" in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Auserhalb der Zeit von 23:00 Uhr bis 04:00 Uhr ist diese Geschichte nur nach einer erfolgreichen Altersverifikation zuganglich.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"/"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        head = soup.find('div', {'class' : 'story-left'})
-        a = head.find('a')
-        self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',stripHTML(a))
-
-        # Find the chapters:
-        for chapter in soup.find('select').findAll('option'):
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        self.story.setMetadata('language','German')
-
-        #find metadata on the story page
-        headtext = stripHTML(head)
-        self.story.setMetadata('datePublished', makeDate(headtext.split('erstellt: ')[1].split('\n')[0], self.dateformat))
-        
-        self.story.setMetadata('dateUpdated', makeDate(headtext.split('aktualisiert: ')[1].split('\n')[0], self.dateformat))
-
-        # second colspan=3 td in head.
-        genres=stripHTML(head.findAll('td',{'colspan':'3'})[1])
-        self.story.extendList('genre',genres[:genres.index('(')].split(', '))
-        # for genre in head.text.split('   ')[3].split('/')[0].split(', '):
-        #     self.story.addToList('genre',genre)
-            
-        if 'fertiggestellt' in headtext:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In Progress')
-
-        #find metadata on the author's page
-        asoup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
-        tr=asoup.findAll('tr')
-        for i in range(1,len(tr)):
-            a = tr[i].find('a')
-            if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
-                break
-        self.setDescription(url,a['onmouseover'].split("', '")[1])
-        
-        td = tr[i].findAll('td')
-        self.story.addToList('category',stripHTML(td[2]))
-        self.story.setMetadata('rating', stripHTML(td[5]))
-        self.story.setMetadata('numWords', stripHTML(td[6]))
- 
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffde has "floodlock" protection
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'storytext'})
-        for a in div.findAll('script'):
-            a.extract()
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fannation.py

@@ -1,44 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2014 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import re
-from base_efiction_adapter import BaseEfictionAdapter
-
-class FanNationAdapter(BaseEfictionAdapter):
-
-    @staticmethod
-    def getSiteDomain():
-        return 'fannation.shades-of-moonlight.com'
-
-    @classmethod
-    def getPathToArchive(self):
-        return '/archive'
-
-    @classmethod
-    def getSiteAbbrev(self):
-        return 'fannation'
-
-    def handleMetadataPair(self, key, value):
-        if key == 'Romance':
-            for val in re.split("\s*,\s*", value):
-                self.story.addToList('romance', val)
-        else:
-            super(FanNationAdapter, self).handleMetadataPair(key, value)
-
-def getClass():
-    return FanNationAdapter

fanficdownloader/adapters/adapter_fhsarchivecom.py

@@ -1,57 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2014 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import re
-from base_efiction_adapter import BaseEfictionAdapter
-
-class FHSArchiveComAdapter(BaseEfictionAdapter):
-
-    @staticmethod
-    def getSiteDomain():
-        return 'fhsarchive.com'
-
-    @classmethod
-    def getPathToArchive(self):
-        return '/autoarchive'
-
-    @classmethod
-    def getSiteAbbrev(self):
-        return 'fhsa'
-        
-    @classmethod
-    def getDateFormat(self):
-        return "%m/%d/%y" 
-   
-    def handleMetadataPair(self, key, value):
-        if key == 'Warnings':
-            for val in re.split("\s*,\s*", value):
-                if value == 'None':
-                    return
-                else:
-                    # toss numbers only.
-                    self.story.addToList('warnings', filter(lambda x : not x.isdigit() , val))
-     
-    #     elif 'Categories' in key:
-    #         for val in re.split("\s*>\s*", value):
-    #             self.story.addToList('category', val)
-        else:
-            super(FHSArchiveComAdapter, self).handleMetadataPair(key, value) 
-                           
-def getClass():
-    return FHSArchiveComAdapter
-    

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -18,7 +18,6 @@
 import time
 import datetime
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 from .. import translit
@@ -34,8 +33,6 @@ def getClass():
     return FicBookNetAdapter
 
 
-logger = logging.getLogger(__name__)
-
 class FicBookNetAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -52,7 +49,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
@@ -69,9 +66,8 @@ class FicBookNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.ficbook.net'
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/readfic/12345 http://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/readfic/12345"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
@@ -79,7 +75,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
         url=self.url
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
         try:
             data = self._fetchUrl(url)
         except urllib2.HTTPError, e:
@@ -98,15 +94,15 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',stripHTML(a))
-        logger.debug("Title: (%s)"%self.story.getMetadata('title'))
+        self.story.setMetadata('title',a.string)
+        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
         
         # Find authorid and URL from... author url.
         a = table.find('a')
         self.story.setMetadata('authorId',a.text) # Author's name is unique
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.text)
-        logger.debug("Author: (%s)"%self.story.getMetadata('author'))
+        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
 
         # Find the chapters:
         chapters = soup.find('div', {'class' : 'part_list'})
@@ -127,7 +123,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
             pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
             update=pubdate
 
-        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
 
         if not ',' in pubdate:
             pubdate=datetime.date.today().strftime(self.dateformat)
@@ -205,13 +201,13 @@ class FicBookNetAdapter(BaseSiteAdapter):
                 break
 				
         summary=soup.find('span', {'class' : 'urlize'})
-        self.setDescription(url,summary)
+        self.setDescription(url,summary.text)
         #self.story.setMetadata('description', summary.text)
             
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -17,7 +17,6 @@
 
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -38,6 +37,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
+        self.story.addToList("category","Harry Potter")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query correct
@@ -45,7 +45,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if m:
             self.story.setMetadata('authorId',m.group('auth'))
             self.story.setMetadata('storyId',m.group('id'))
-            
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL(url)
         else:
@@ -57,9 +57,8 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.fictionalley.org'
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/authors/drt/DA.html http://"+cls.getSiteDomain()+"/authors/drt/JOTP01a.html"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/authors/drt/DA.html http://"+self.getSiteDomain()+"/authors/drt/JOTP01a.html"
 
     def getSiteURLPattern(self):
         # http://www.fictionalley.org/authors/drt/DA.html
@@ -70,7 +69,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if self.is_adult or self.getConfig("is_adult"):
             params={'iamold':'Yes',
                     'action':'ageanswer'}
-            logger.info("Attempting to get cookie for %s" % url)
+            logging.info("Attempting to get cookie for %s" % url)
             ## posting on list doesn't work, but doesn't hurt, either.
             data = self._postUrl(url,params)
         else:
@@ -81,7 +80,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
         ## could be either chapter list page or one-shot text page.
         url = self.url
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._postFetchWithIAmOld(url)
@@ -109,9 +108,9 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             storya = soup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
             self._setURL(storya['href'])
             url=self.url
-            logger.debug("Normalizing to URL: "+url)
+            logging.debug("Normalizing to URL: "+url)
             ## title's right there...
-            self.story.setMetadata('title',stripHTML(storya))
+            self.story.setMetadata('title',storya.string)
             data = self._fetchUrl(url)
             soup = bs.BeautifulSoup(data)
             chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
@@ -121,7 +120,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             ## same way.
             chapsoup = bs.BeautifulSoup(chapterdata)
             storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self.story.setMetadata('title',stripHTML(storya))
+            self.story.setMetadata('title',storya.string)
             del chapsoup
 
         del chapterdata
@@ -195,7 +194,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
 	# find <!-- headerend --> & <!-- footerstart --> and
@@ -224,16 +223,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             
         if not data or not text:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        # not sure how, but we can get html, etc tags still in some
-        # stories.  That breaks later updates because it confuses
-        # epubutils.py
-        for tag in text.findAll('head'):
-            tag.extract()
-        
-        for tag in text.findAll('body') + text.findAll('html'):
-            tag.name = 'div'
-            
+    
         return self.utf8FromSoup(url,text)
 
 def getClass():

fanficdownloader/adapters/adapter_fictionmaniatv.py

@@ -1,178 +0,0 @@
-import re
-import urllib2
-import urlparse
-
-from .. import BeautifulSoup
-from ..BeautifulSoup import NavigableString
-
-from base_adapter import BaseSiteAdapter, makeDate
-from .. import exceptions
-
-
-def getClass():
-    return FictionManiaTVAdapter
-
-
-def _get_query_data(url):
-    components = urlparse.urlparse(url)
-    query_data = urlparse.parse_qs(components.query)
-    return dict((key, data[0]) for key, data in query_data.items())
-
-# yields Tag _and_ NavigableString siblings from the given tag. The
-# BeautifulSoup findNextSiblings() method for some reasons only returns either
-# NavigableStrings _or_ Tag objects, not both.
-def _yield_next_siblings(tag):
-    sibling = tag.nextSibling
-    while sibling:
-        yield sibling
-        sibling = sibling.nextSibling
-
-
-class FictionManiaTVAdapter(BaseSiteAdapter):
-    SITE_ABBREVIATION = 'fmt'
-    SITE_DOMAIN = 'fictionmania.tv'
-
-    BASE_URL = 'http://' + SITE_DOMAIN + '/stories/'
-    READ_TEXT_STORY_URL_TEMPLATE = BASE_URL + 'readtextstory.html?storyID=%s'
-    DETAILS_URL_TEMPLATE = BASE_URL + 'details.html?storyID=%s'
-
-    DATETIME_FORMAT = '%m/%d/%Y'
-    ALTERNATIVE_DATETIME_FORMAT = '%m/%d/%y'
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        query_data = urlparse.parse_qs(self.parsedUrl.query)
-        story_id = query_data['storyID'][0]
-
-        self.story.setMetadata('storyId', story_id)
-        self._setURL(self.READ_TEXT_STORY_URL_TEMPLATE % story_id)
-        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
-
-        # Always single chapters, probably should use the Anthology feature to
-        # merge chapters of a story
-        self.story.setMetadata('numChapters', 1)
-
-    def _customized_fetch_url(self, url, exception=None, parameters=None):
-        if exception:
-            try:
-                data = self._fetchUrl(url, parameters)
-            except urllib2.HTTPError:
-                raise exception(self.url)
-        # Just let self._fetchUrl throw the exception, don't catch and
-        # customize it.
-        else:
-            data = self._fetchUrl(url, parameters)
-
-        return BeautifulSoup.BeautifulSoup(data)
-
-    @staticmethod
-    def getSiteDomain():
-        return FictionManiaTVAdapter.SITE_DOMAIN
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return cls.READ_TEXT_STORY_URL_TEMPLATE % 1234
-
-    def getSiteURLPattern(self):
-        return re.escape(self.BASE_URL) + '(readtextstory|details)\.html\?storyID=\d+$'
-
-    def extractChapterUrlsAndMetadata(self):
-        url = self.DETAILS_URL_TEMPLATE % self.story.getMetadata('storyId')
-        soup = self._customized_fetch_url(url)
-
-        keep_summary_html = self.getConfig('keep_summary_html')
-        for row in soup.find('table')('tr'):
-            cells = row('td')
-            key = cells[0].b.string.strip(':')
-            try:
-                value = cells[1].string
-            except AttributeError:
-                value = None
-
-            if key == 'Story Name-Title':
-                self.story.setMetadata('title', value)
-                self.chapterUrls.append((value, self.url))
-
-            elif key == 'File Name':
-                self.story.setMetadata('fileName', value)
-
-            elif key == 'File Size':
-                self.story.setMetadata('fileSize', value)
-
-            elif key == 'Author':
-                element = cells[1].a
-                self.story.setMetadata('author', element.string)
-                query_data = _get_query_data(element['href'])
-                self.story.setMetadata('authorId', query_data['word'])
-                self.story.setMetadata('authorUrl', urlparse.urljoin(url, element['href']))
-
-            elif key == 'Date Added':
-                try:
-                    date = makeDate(value, self.DATETIME_FORMAT)
-                except ValueError:
-                    date = makeDate(value, self.ALTERNATIVE_DATETIME_FORMAT)
-                self.story.setMetadata('datePublished', date)
-
-            elif key == 'Old Name':
-                self.story.setMetadata('oldName', value)
-
-            elif key == 'New Name':
-                self.story.setMetadata('newName', value)
-
-            elif key == 'Other Key Names':
-                for name in value.split(', '):
-                    self.story.addToList('characters', name)
-
-            # I have no clue how the rating system works, if you are reading
-            # transgender fanfiction, you are probably an adult.
-            elif key == 'Rating':
-                self.story.setMetadata('rating', value)
-
-            elif key == 'Complete':
-                self.story.setMetadata('status', 'Complete' if value == 'Complete' else 'In-Progress')
-
-            elif key == 'Categories':
-                for element in cells[1]('a'):
-                    self.story.addToList('category', element.string)
-
-            elif key == 'Key Words':
-                for element in cells[1]('a'):
-                    self.story.addToList('keyWords', element.string)
-
-            elif key == 'Main Characters Age':
-                element = cells[1].a
-                self.story.setMetadata('mainCharactersAge', element.string)
-
-            elif key == 'Synopsis':
-                element = cells[1]
-
-                # Replace td with div to avoid possible strange formatting in
-                # the ebook later on
-                element.name = 'div'
-
-                if keep_summary_html:
-                    self.story.setMetadata('description', unicode(element))
-                else:
-                    self.story.setMetadata('description', ''.join(element(text=True)))
-
-            elif key == 'Reads':
-                self.story.setMetadata('readings', value)
-
-    def getChapterText(self, url):
-        soup = self._customized_fetch_url(url)
-        element = soup.find('pre')
-        element.name = 'div'
-
-        # The story's content is contained in a <pre> tag, probably taken 1:1
-        # from the source text file. A simple replacement of all newline
-        # characters with a break line tag should take care of formatting.
-
-        # While wrapping in paragraphs would be possible, it's too much work,
-        # I'd rather display the story 1:1 like it was found in the pre tag.
-        content = unicode(element)
-        content = content.replace('\n', '<br />')
-
-        if self.getConfig('non_breaking_spaces'):
-            content = content.replace(' ', '&nbsp;')
-        return content

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -1,194 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import time
-import json
-
-from .. import BeautifulSoup as bs
-#from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class FictionPadSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','fpad')
-        self.dateformat = "%Y-%m-%dT%H:%M:%SZ"
-        self.is_adult=False
-        self.username = None
-        self.password = None
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-            
-            # normalized story URL.
-            self._setURL("https://"+self.getSiteDomain()
-                         +"/author/"+m.group('author')
-                         +"/stories/"+self.story.getMetadata('storyId'))
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-
-    @staticmethod
-    def getSiteDomain():
-        return 'fictionpad.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://fictionpad.com/author/Author/stories/1234/Some-Title"
-    
-    def getSiteURLPattern(self):
-        # http://fictionpad.com/author/Serdd/stories/4275
-        return r"http(s)?://(www\.)?fictionpad\.com/author/(?P<author>[^/]+)/stories/(?P<id>\d+)"
-
-# <form method="post" action="/signin">
-#     <input name="authenticity_token" type="hidden" value="u+cfdXh46dRnwVnSlmE2B2BFmHgu760paqgBG6KQeos=" />
-#     <input type="hidden" name="remember" value="1">
-#     <strong class="help-start text-center">or with FictionPad</strong>
-#     <label class="control-label hidden-placeholder">Pseudonym or Email Address</label>
-#     <input name="login" class="input-block-level" type="text" placeholder="Pseudonym or Email Address" maxlength="50" required autofocus>
-#     <label class="control-label hidden-placeholder">Password</label>
-#     <input name="password" class="input-block-level" type="password" placeholder="Password" minlength="6" required>
-#     <button type="submit" class="btn btn-primary btn-block">Sign In</button>
-#     <p class="help-end">
-#         <a href="/passwordreset">Forgot your password?</a>
-#     </p>
-# </form>
-    def performLogin(self):
-        params = {}
-
-        if self.password:
-            params['login'] = self.username
-            params['password'] = self.password
-        else:
-            params['login'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['remember'] = '1'
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/signin'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['login']))
-
-        ## need to pull empty login page first to get authenticity_token
-        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
-        params['authenticity_token']=soup.find('input', {'name':'authenticity_token'})['value']
-        
-        data = self._postUrl(loginUrl, params)
-
-        if "Invalid email/pseudonym and password combination." in data:
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['login']))
-            raise exceptions.FailedToLogin(loginUrl,params['login'])
-            
-
-    def extractChapterUrlsAndMetadata(self):
-        # fetch the chapter.  From that we will get almost all the
-        # metadata and chapter list
-
-        url=self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-            if "This is a mature story.  Please sign in to read it." in data:
-                self.performLogin()
-                data = self._fetchUrl(url)
-            
-            find = "wordyarn.config.page = "
-            data = data[data.index(find)+len(find):]
-            data = data[:data.index("</script>")]
-            data = data[:data.rindex(";")]
-            data = data.replace('tables:','"tables":')
-            tables = json.loads(data)['tables']
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(url)
-            else:
-                raise e
-
-        # looks like only one author per story allowed.
-        author = tables['users'][0]
-        story = tables['stories'][0]
-        story_ver = tables['story_versions'][0]
-        print("story:%s"%story)
-        
-        self.story.setMetadata('authorId',author['id'])
-        self.story.setMetadata('author',author['display_name'])
-        self.story.setMetadata('authorUrl','https://'+self.host+'/author/'+author['display_name']+'/stories')
-
-        self.story.setMetadata('title',story_ver['title'])
-        self.setDescription(url,story_ver['description'])
-        
-        if not ('assets/story_versions/covers' in story_ver['profile_image_url@2x']):
-            self.setCoverImage(url,story_ver['profile_image_url@2x'])
-
-        self.story.setMetadata('datePublished',makeDate(story['published_at'], self.dateformat))
-        self.story.setMetadata('dateUpdated',makeDate(story['published_at'], self.dateformat))
-
-        self.story.setMetadata('followers',story['followers_count'])
-        self.story.setMetadata('comments',story['comments_count'])
-        self.story.setMetadata('views',story['views_count'])
-        self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
-        if 'dislikes' in story:
-            self.story.setMetadata('dislikes',int(story['dislikes']))
-
-        if story_ver['is_complete']:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        self.story.setMetadata('rating', story_ver['maturity_level'])
-        self.story.setMetadata('numWords', unicode(story_ver['word_count']))
-
-        for i in tables['fandoms']:
-            self.story.addToList('category',i['name'])
-        
-        for i in tables['genres']:
-            self.story.addToList('genre',i['name'])
-        
-        for i in tables['characters']:
-            self.story.addToList('characters',i['name'])
-
-        for c in tables['chapters']:
-            chtitle = "Chapter %d"%c['number']
-            if c['title']:
-                chtitle += " - %s"%c['title']
-            self.chapterUrls.append((chtitle,c['body_url']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-            
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        if not url:
-            data = u"<em>This chapter has no text.</em>"
-        else:
-            data = self._fetchUrl(url)
-        soup = bs.BeautifulSoup(u"<div id='story'>"+data+u"</div>")
-        return self.utf8FromSoup(url,soup)
-    
-def getClass():
-    return FictionPadSiteAdapter
-

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -17,7 +17,6 @@
 
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time
@@ -39,12 +38,11 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.fictionpress.com','m.fictionpress.com']
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
+    def getSiteExampleURLs(self):
+        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title"
 
     def getSiteURLPattern(self):
-        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
 
 def getClass():
     return FictionPressComSiteAdapter

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -17,7 +17,6 @@
 
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time
@@ -43,11 +42,10 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     @staticmethod
     def getSiteDomain():
-        return 'ficwad.com'
+        return 'www.ficwad.com'
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://ficwad.com/story/1234"
+    def getSiteExampleURLs(self):
+        return "http://www.ficwad.com/story/137169"
 
     def getSiteURLPattern(self):
         return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"
@@ -63,59 +61,42 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             params['password'] = self.getConfig("password")
 
         loginUrl = 'http://' + self.getSiteDomain() + '/account/login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['username']))
-        d = self._postUrl(loginUrl,params,usecache=False)
+        d = self._postUrl(loginUrl,params)
 
         if "Login attempt failed..." in d:
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['username']))
             raise exceptions.FailedToLogin(url,params['username'])
             return False
         else:
             return True        
     
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
-
     def extractChapterUrlsAndMetadata(self):
 
         # fetch the chapter.  From that we will get almost all the
         # metadata and chapter list
 
         url = self.url
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
-            data = self._fetchUrl(url)
-            # non-existent/removed story urls get thrown to the front page.
-            if "<h2>Welcome to FicWad</h2>" in data:
-                raise exceptions.StoryDoesNotExist(self.url)
-            soup = bs.BeautifulSoup(data)
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
             
-        # if blocked, attempt login.
-        if soup.find("div",{"class":"blocked"}):
-            if self.performLogin(url): # performLogin raises
-                                       # FailedToLogin if it fails.
-                soup = bs.BeautifulSoup(self._fetchUrl(url,usecache=False))
-
-        divstory = soup.find('div',id='story')
-        storya = divstory.find('a',href=re.compile("^/story/\d+$"))
-        if storya : # if there's a story link in the divstory header, this is a chapter page.
+        h3 = soup.find('h3')
+        storya = h3.find('a',href=re.compile("^/story/\d+$"))
+        if storya : # if there's a story link in the h3 header, this is a chapter page.
             # normalize story URL on chapter list.
             self.story.setMetadata('storyId',storya['href'].split('/',)[2])
             url = "http://"+self.getSiteDomain()+storya['href']
-            logger.debug("Normalizing to URL: "+url)
+            logging.debug("Normalizing to URL: "+url)
             self._setURL(url)
             try:
                 soup = bs.BeautifulSoup(self._fetchUrl(url))
@@ -126,24 +107,24 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                     raise e
 
         # if blocked, attempt login.
-        if soup.find("div",{"class":"blocked"}):
+        if soup.find("li",{"class":"blocked"}):
             if self.performLogin(url): # performLogin raises
                                        # FailedToLogin if it fails.
-                soup = bs.BeautifulSoup(self._fetchUrl(url,usecache=False))
+                soup = bs.BeautifulSoup(self._fetchUrl(url))
 
         # title - first h4 tag will be title.
-        titleh4 = soup.find('div',{'class':'storylist'}).find('h4')
-        self.story.setMetadata('title', stripHTML(titleh4.a))
+        titleh4 = soup.find('h4')
+        self.story.setMetadata('title', titleh4.a.string)
 
         # Find authorid and URL from... author url.
-        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+"))
+        a = soup.find('a', href=re.compile(r"^/author/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
 
         # description
         storydiv = soup.find("div",{"id":"story"})
-        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p)
+        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p.string)
         #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
 
         # most of the meta data is here:
@@ -152,7 +133,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # warnings
         # <span class="req"><a href="/help/38" title="Medium Spoilers">[!!] </a> <a href="/help/38" title="Rape/Sexual Violence">[R] </a> <a href="/help/38" title="Violence">[V] </a> <a href="/help/38" title="Child/Underage Sex">[Y] </a></span>
-        spanreq = metap.find("span",{"class":"story-warnings"})
+        spanreq = metap.find("span",{"class":"req"})
         if spanreq: # can be no warnings.
             for a in spanreq.findAll("a"):
                 self.story.addToList('warnings',a['title'])
@@ -178,16 +159,16 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                 if g:
                     self.story.addToList('characters',g)
         
-        m = re.match(r".*?Published: ([0-9-]+?) -.*?",metastr)
+        m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
         if m:
-            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y-%m-%d"))
+            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))
 
         # Updated can have more than one space after it. <shrug>
-        m = re.match(r".*?Updated: ([0-9-]+?) +-.*?",metastr) 
+        m = re.match(r".*?Updated: ([0-9/]+?) +-.*?",metastr) 
         if m:
-            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y-%m-%d"))
+            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y/%m/%d"))
 
-        m = re.match(r".*? - ([0-9,]+?) words.*?",metastr)
+        m = re.match(r".*? - ([0-9/]+?) words.*?",metastr)
         if m:
             self.story.setMetadata('numWords',m.group(1))
 
@@ -198,7 +179,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # get the chapter list first this time because that's how we
         # detect the need to login.
-        storylistul = soup.find('ul',{'class':'storylist'})
+        storylistul = soup.find('ul',{'id':'storylist'})
         if not storylistul:
             # no list found, so it's a one-chapter story.
             self.chapterUrls.append((self.story.getMetadata('title'),url))
@@ -220,7 +201,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
 
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -17,10 +17,10 @@
 
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
+from datetime import datetime
 import json
 
 from .. import BeautifulSoup as bs
@@ -41,10 +41,6 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
         self.is_adult = False
         
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-            
     @staticmethod
     def getSiteDomain():
         return 'www.fimfiction.net'
@@ -54,23 +50,16 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway
         return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com']
 
-    @classmethod
-    def getSiteExampleURLs(cls):
+    def getSiteExampleURLs(self):
         return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
 
     def getSiteURLPattern(self):
-        return r"https?://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
+        return r"http://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
         
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
-    
-    def doExtractChapterUrlsAndMetadata(self,get_cover=True):
+    def extractChapterUrlsAndMetadata(self):
         
         if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
             cookie = cl.Cookie(version=0, name='view_mature', value='true',
                                port=None, port_specified=False,
                                domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -82,17 +71,15 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                                comment_url=None,
                                rest={'HttpOnly': None},
                                rfc2109=False)
-            self.cookiejar.set_cookie(cookie)
-
-        ##---------------------------------------------------------------------------------------------------
-        ## Get the story's title page. Check if it exists.
-
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)
+        
         try:
-            # don't use cache if manual is_adult--should only happen
-            # if it's an adult story and they don't have is_adult in ini.
-            data = self.do_fix_blockquotes(self._fetchUrl(self.url,
-                                                          usecache=(not self.is_adult)))
-            soup = bs.BeautifulSoup(data)
+            apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
+            apiData = json.loads(apiResponse)
+            
+            # Unfortunately, we still need to load the story index page to parse the characters
+            data = self._fetchUrl(self.url)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -102,237 +89,80 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
+        if "/images/missing_story.png" in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+        
+        if "Invalid story id" in apiData.values():
+            raise exceptions.StoryDoesNotExist(self.url)
+        
+        if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
         if self.password:
             params = {}
             params['password'] = self.password
-            data = self._postUrl(self.url, params)
-            soup = bs.BeautifulSoup(data)
+            data = self._postUrl(self.url,params)
 
-        if not (soup.find('form', {'id' : 'password_form'}) == None):
+        if "Enter the password the author set for this story to view it." in data:
             if self.getConfig('fail_on_password'):
                 raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
             else:
                 raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
-
-        ##----------------------------------------------------------------------------------------------------
-        ## Extract metadata
-
-        storyContentBox = soup.find('div', {'class':'story_content_box'})
-
-        # Title
-        title = storyContentBox.find('a', {'class':re.compile(r'.*\bstory_name\b.*')})
-        self.story.setMetadata('title',stripHTML(title))
-
-        # Author
-        author = storyContentBox.find('div', {'class':'author'}).find('a')
-        self.story.setMetadata("author", stripHTML(author))
-        #No longer seems to be a way to access Fimfiction's internal author ID
-        self.story.setMetadata("authorId", self.story.getMetadata("author"))
-        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), stripHTML(author)))
-
-        #Rating text is replaced with full words for historical compatibility after the site changed
-        #on 2014-10-27
-        rating = stripHTML(storyContentBox.find('a', {'class':re.compile(r'.*\bcontent-rating-.*')}))
-        rating = rating.replace("E", "Everyone").replace("T", "Teen").replace("M", "Mature")
-        self.story.setMetadata("rating", rating)
-
-        # Chapters
-        for chapter in storyContentBox.findAll('a',{'class':'chapter_link'}):
-            self.chapterUrls.append((stripHTML(chapter), 'http://'+self.host+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # Status
-        # In the case of Fimfiction, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
-        # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
-        # and 'Complete' becomes 'Completed'. 'Cancelled' and 'On Hiatus' are passed through, it's easy now for users
-        # to change/remove if they want with replace_metadata
-        status = stripHTML(storyContentBox.find('span', {'class':re.compile(r'.*\bcompleted-status-.*')}))
-        status = status.replace("Incomplete", "In-Progress").replace("Complete", "Completed")
-        self.story.setMetadata("status", status)
-
-        # Genres and Warnings
-        # warnings were folded into general categories in the 2014-10-27 site update
-        categories = storyContentBox.findAll('a', {'class':re.compile(r'.*\bstory_category\b.*')})
-        for category in categories:
-            category = stripHTML(category)
-            if category == "Gore" or category == "Sex":
-                self.story.addToList('warnings', category)
-            else:
-                self.story.addToList('genre', category)
-
-        # Word count
-        wordCountText = stripHTML(storyContentBox.find('li', {'class':'bottom'}).find('div', {'class':'word_count'}))
-        self.story.setMetadata("numWords", re.sub(r'[^0-9]', '', wordCountText))
-
-        # Cover image
-        storyImage = storyContentBox.find('div', {'class':'story_image'})
-        if storyImage:
-            coverurl = storyImage.find('a')['href']
-            if coverurl.startswith('//'): # fix for img urls missing 'http:'
-                coverurl = "http:"+coverurl
-            if get_cover:
-                self.setCoverImage(self.url,coverurl)
-
-            coverSource = storyImage.find('a', {'class':'source'})
-            if coverSource:
-                self.story.setMetadata('coverSourceUrl', coverSource['href'])
-                #There's no text associated with the cover source link, so just
-                #reuse the URL. Makes it clear it's an external link leading
-                #outside of the fanfic site, at least.
-                self.story.setMetadata('coverSource', coverSource['href'])
-
-        # fimf has started including extra stuff inside the description div.
-        descdivstr = u"%s"%storyContentBox.find("div", {"class":"description"})
-        hrstr=u"<hr />"
-        descdivstr = u'<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
-        self.setDescription(self.url,descdivstr)
-
-        # Find the newest and oldest chapter dates
-        storyData = storyContentBox.find('div', {'class':'story_data'})
-        oldestChapter = None
-        newestChapter = None
-        self.newestChapterNum = None # save for comparing during update.
-        # Scan all chapters to find the oldest and newest, on
-        # FiMFiction it's possible for authors to insert new chapters
-        # out-of-order or change the dates of earlier ones by editing
-        # them--That WILL break epub update.
-        for index, chapterDate in enumerate(storyData.findAll('span', {'class':'date'})):
-            chapterDate = self.ordinal_date_string_to_date(chapterDate.contents[1])
-            if oldestChapter == None or chapterDate < oldestChapter:
-                oldestChapter = chapterDate
-            if newestChapter == None or chapterDate > newestChapter:
-                newestChapter = chapterDate
-                self.newestChapterNum = index
-
-        if newestChapter is None:
-            #this will only be true when updating metadata for stories that have 0 chapters
-            #there is a "last modified" date given on the page, extract it and use that.
-            moddatetag = storyContentBox.find('span', {'class':'last_modified'})
-            if not moddatetag is None:
-                newestChapter = self.ordinal_date_string_to_date(moddatetag('span')[1].text)
-
-        # Date updated
-        self.story.setMetadata("dateUpdated", newestChapter)
-
-        # Date published
-        # falls back to oldest chapter date for stories that haven't been officially published yet
-        pubdatetag = storyContentBox.find('span', {'class':'date_approved'})
-        if pubdatetag is None:
-            if oldestChapter is None:
-                #this will only be true when updating metadata for stories that have 0 chapters
-                #and that have never been officially published - a rare occurrence. Fall back to last
-                #modified date as the publication date, it's all that we've got.
-                self.story.setMetadata("datePublished", newestChapter)
-            else:
-                self.story.setMetadata("datePublished", oldestChapter)
-        else:
-            pubDate = self.ordinal_date_string_to_date(pubdatetag('span')[1].text)
-            self.story.setMetadata("datePublished", pubDate)
-
-        # Characters
-        chars = storyContentBox.find("div", {"class":"extra_story_data"})
-        for character in chars.findAll("a", {"class":"character_icon"}):
-            self.story.addToList("characters", character['title'])
-
-        # Likes and dislikes
-        storyToolbar = soup.find('div', {'class':'story-toolbar'})
-        likes = storyToolbar.find('span', {'class':'likes'})
-        if not likes is None:
-            self.story.setMetadata("likes", stripHTML(likes))
-        dislikes = storyToolbar.find('span', {'class':'dislikes'})
-        if not dislikes is None:
-            self.story.setMetadata("dislikes", stripHTML(dislikes))
-
-        # Highest view for a chapter and total views
-        viewSpan = storyToolbar.find('span', {'title':re.compile(r'.*\btotal views\b.*')})
-        self.story.setMetadata("views", re.sub(r'[^0-9]', '', stripHTML(viewSpan)))
-        self.story.setMetadata("total_views", re.sub(r'[^0-9]', '', viewSpan['title']))
-
-        # Comment count
-        commentSpan = storyToolbar.find('span', {'title':re.compile(r'.*\bcomments\b.*')})
-        self.story.setMetadata("comment_count", re.sub(r'[^0-9]', '', stripHTML(commentSpan)))
-
-        # Short description
-        descriptionMeta = soup.find('meta', {'property':'og:description'})
-        self.story.setMetadata("short_description", stripHTML(descriptionMeta['content']))
-
-        #groups
-        if soup.find('button', {'id':'button-view-all-groups'}):
-            groupResponse = self._fetchUrl("http://www.fimfiction.net/ajax/groups/story_groups_list.php?story=%s" % (self.story.getMetadata("storyId")))
-            groupData = json.loads(groupResponse)
-            groupList = bs.BeautifulSoup(groupData["content"])
-        else:
-            groupList = soup.find('ul', {'id':'story-groups-list'})
-
-        if not (groupList == None):
-            for groupName in groupList.findAll('a'):
-                self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
-                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
-
-        #sequels
-        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
-        if not sequelStoryHeader == None:
-            sequelContainer = sequelStoryHeader.parent.parent
-            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
-                self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
-                self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
-                
-        #The link to the prequel is embedded in the description text, so erring
-        #on the side of caution and wrapping this whole thing in a try block.
-        #If anything goes wrong this probably wasn't a valid prequel link.
-        try:
-            description = soup.find('div', {'class':'description'})
-            firstHR = description.find("hr")
-            nextSib = firstHR.nextSibling
-            if "This story is a sequel to" in nextSib.string:
-                link = nextSib.nextSibling
-                if link.name == "a":
-                    self.story.setMetadata("prequelUrl", 'http://'+self.host+link["href"])
-                    self.story.setMetadata("prequel", stripHTML(link))
-        except:
-            pass
-
-    def ordinal_date_string_to_date(self, datestring):
-        datestripped=re.sub(r"(\d+)(st|nd|rd|th)", r"\1", datestring.strip())
-        return makeDate(datestripped, self.dateformat)
-
-    def hookForUpdates(self,chaptercount):
-        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
-            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
-            self.oldchapters = self.oldchapters[:self.newestChapterNum]
-        return len(self.oldchapters)
-
-    def do_fix_blockquotes(self,data):
-        if self.getConfig('fix_fimf_blockquotes'):
-            # <p class="double"><blockquote>
-            # </blockquote></p>
-            # include > in re groups so there's always something in the group.
-            data = re.sub(r'<p([^>]*>\s*)<blockquote([^>]*>)',r'<blockquote\2<p\1',data)
-            data = re.sub(r'</blockquote(>\s*)</p>',r'</p\1</blockquote>',data)
-        return data
+         
+        storyMetadata = apiData["story"]    
+            
+        self.story.setMetadata("title", storyMetadata["title"])
+        self.story.setMetadata("author", storyMetadata["author"]["name"])
+        self.story.setMetadata("authorId", storyMetadata["author"]["id"])
+        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
         
+        chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
+        for chapter in chapters:
+            self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
+        self.story.setMetadata("numChapters", len(self.chapterUrls))
+        
+        # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
+        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
+        # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
+        # status to leave unchanged.
+        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
+        self.story.setMetadata("status", status)
+        self.story.setMetadata("rating", storyMetadata["content_rating_text"])
+            
+        for category in storyMetadata["categories"]:
+            if storyMetadata["categories"][category]:
+                self.story.addToList("genre", category) 
+
+        self.story.addToList("category", "My Little Pony")
+        
+        self.story.setMetadata("numWords", str(storyMetadata["words"]))
+        
+        
+        # fimfic is the first site with an explicit cover image.
+        if self.getConfig('include_images') and "image" in storyMetadata.keys():
+            coverurl = storyMetadata["image"]
+            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
+                coverurl = "http:"+coverurl
+            self.story.addImgUrl(self,self.url,coverurl,self._fetchUrlRaw,cover=True)
+            
+        self.setDescription(self.url, storyMetadata["description"])
+        
+        # Dates are in Unix time
+        # Take the publish date from the first chapter posted
+        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        rawDateUpdated = storyMetadata["date_modified"]
+        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
+        for character in [character_icon["title"] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
+            self.story.addToList("characters", character)
+            
+            
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data)
-        if not (soup.find('form', {'id' : 'password_form'}) == None):
-            if self.password:
-                params = {}
-                params['password'] = self.password
-                data = self._postUrl(url, params)
-            else:
-                print("Chapter %s needed password but no password was present" % url)
-
-        data = self.do_fix_blockquotes(data)
-
-        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
+        logging.debug('Getting chapter text from: %s' % url)
+        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
         return self.utf8FromSoup(url,soup)
+        

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -1,288 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return FineStoriesComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class FineStoriesComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
-        if 'storyInfo' in self.story.getMetadata('storyId'):
-            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','fnst')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y-%m-%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'finestories.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/s/1234 http://"+cls.getSiteDomain()+"/s/1234:4010 http://"+cls.getSiteDomain()+"/library/storyInfo.php?id=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Free Registration' in data \
-                or "Invalid Password!" in data \
-                or "Invalid User Name!" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['theusername'] = self.username
-            params['thepassword'] = self.password
-        else:
-            params['theusername'] = self.getConfig("username")
-            params['thepassword'] = self.getConfig("password")
-        params['rememberMe'] = '1'
-        params['page'] = 'http://'+self.getSiteDomain()+'/'
-        params['submit'] = 'Login'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['theusername']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "My Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['theusername']))
-            raise exceptions.FailedToLogin(url,params['theusername'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"/a/\w+"))
-        self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',a.text)
-
-        # Find the chapters:
-        chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$"))
-        if len(chapters) != 0:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # surprisingly, the detailed page does not give enough details, so go to author's page
-        
-        skip=0
-        i=0
-        while i == 0:
-            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
-        
-            a = asoup.findAll('td', {'class' : 'lc2'})
-            for lc2 in a:
-                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
-                    i=1
-                    break
-                if a[len(a)-1] == lc2:
-                    skip=skip+10
-        
-        for cat in lc2.findAll('div', {'class' : 'typediv'}):
-            self.story.addToList('category',cat.text)
-        
-        self.story.setMetadata('numWords', lc2.findNext('td', {'class' : 'num'}).text)
-        
-        lc4 = lc2.findNext('td', {'class' : 'lc4'})
-        
-        
-        try:
-            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
-            i = a.parent.text.split('(')[1].split(')')[0]
-            self.setSeries(a.text, i)
-            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
-        except:
-            pass
-        try:
-            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
-            self.story.addToList("category",a.text)
-        except:
-            pass
-            
-        for a in lc4.findAll('span', {'class' : 'help'}):
-            a.extract()
-            
-        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),lc4.text.split('[More Info')[0])
-            
-        for b in lc4.findAll('b'):
-            label = b.text
-            value = b.nextSibling
-            
-            if 'For Age' in label:
-                self.story.setMetadata('rating', value)
-                
-            if 'Tags' in label:
-                for genre in value.split(', '):
-                    self.story.addToList('genre',genre)
-                    
-            if 'Posted' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
-                
-            if 'Concluded' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
-                
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
-                
-        status = lc4.find('span', {'class' : 'ab'})
-        if  status != None:
-            self.story.setMetadata('status', 'In-Progress')
-            if "Last Activity" in status.text:
-                self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat))
-        else:
-            self.story.setMetadata('status', 'Completed')
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-        
-        # some big chapters are split over several pages
-        pager = div.find('span', {'class' : 'pager'})
-        if pager != None:
-            urls=pager.findAll('a')
-            urls=urls[:len(urls)-1]
-            
-            
-            for ur in urls:
-                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-                div1 = soup.find('div', {'id' : 'story'})
-                
-                # appending next section
-                last=div.findAll('p')
-                next=div1.find('span', {'class' : 'conTag'}).nextSibling
-            
-                last[len(last)-1]=last[len(last)-1].append(next)
-                div.append(div1)
-            
-        # removing all the left-over stuff    
-        for a in div.findAll('span'):
-            a.extract()
-            
-        for a in div.findAll('h1'):
-            a.extract()
-        for a in div.findAll('h2'):
-            a.extract()
-        for a in div.findAll('h3'):
-            a.extract()
-        for a in div.findAll('h4'):
-            a.extract()
-        for a in div.findAll('br'):
-            a.extract()
-        for a in div.findAll('div', {'class' : 'date'}):
-            a.extract()
-            
-        a = div.find('form')
-        if a != None:
-            b = a.nextSibling
-            while b != None:
-                a.extract()
-                a=b
-                b=b.nextSibling
-        
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -1,311 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return GrangerEnchantedCom
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class GrangerEnchantedCom(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        self.section=self.parsedUrl.path.split('/',)[1]
-             
-        # normalized story URL.
-        if "malfoymanor" in self.parsedUrl.netloc:
-            self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId'))
-            self.story.addToList("category","The Manor")
-        else:
-            self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','gech')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%b/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'grangerenchanted.com'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        if "enchant" in self.section:
-            loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
-        else:
-            loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=1"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Read' in label:
-                self.story.setMetadata('read', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-        try:
-            self.story.setMetadata('reviews',
-                                   stripHTML(soup.find('div',{'id':'sort'}).
-                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story1'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -39,11 +37,12 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
+        self.story.addToList("category","Harry Potter")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only psid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
@@ -57,9 +56,8 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com']
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234"
+    def getSiteExampleURLs(self):
+        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234 http://harrypotterfanfiction.com/viewstory.php?psid=5678"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("harrypotterfanfiction.com/viewstory.php?psid=")+r"\d+$"
@@ -75,7 +73,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -93,7 +91,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',stripHTML(a))
+        self.story.setMetadata('title',a.string)
         ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
         if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
@@ -132,59 +130,59 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Finding the metadata is a bit of a pain.  Most of the meta
         ## data is in a center.table without a bgcolor.
-        #for center in soup.findAll('center'):
-        table = soup.find('table',{'class':'storymaininfo'})
-        if table:
-            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-            # Rating: 12+ Story Reviews: 3
-            # Chapters: 3
-            # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-            # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-            # Status: Completed
-            # First Published: 2010.09.02
-            # Last Published Chapter: 2010.09.28
-            # Last Updated: 2010.09.28
-            # Favorite Story Of: 1 users
-            # Warnings: Scenes of a Mild Sexual Nature
+        for center in soup.findAll('center'):
+            table = center.find('table',{'bgcolor':None})
+            if table:
+                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+                # Rating: 12+ Story Reviews: 3
+                # Chapters: 3
+                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
+                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
+                # Status: Completed
+                # First Published: 2010.09.02
+                # Last Published Chapter: 2010.09.28
+                # Last Updated: 2010.09.28
+                # Favorite Story Of: 1 users
+                # Warnings: Scenes of a Mild Sexual Nature
 
-            m = re.match(r".*?Status: Completed.*?",metastr)
-            if m:
-                self.story.setMetadata('status','Completed')
-            else:
-                self.story.setMetadata('status','In-Progress')
+                m = re.match(r".*?Status: Completed.*?",metastr)
+                if m:
+                    self.story.setMetadata('status','Completed')
+                else:
+                    self.story.setMetadata('status','In-Progress')
 
-            m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-            if m:
-                self.story.setMetadata('rating', m.group(1))
+                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
+                if m:
+                    self.story.setMetadata('rating', m.group(1))
 
-            m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-            if m:
-                for g in m.group(1).split(','):
-                    self.story.addToList('genre',g)
-    
-            m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-            if m:
-                for g in m.group(1).split(','):
-                    self.story.addToList('characters',g)
-    
-            m = re.match(r".*?Warnings: (.+).*?",metastr)
-            if m:
-                for w in m.group(1).split(','):
-                    if w != 'Now Warnings':
-                        self.story.addToList('warnings',w)
-    
-            m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-            if m:
-                self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
+                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
+                if m:
+                    for g in m.group(1).split(','):
+                        self.story.addToList('genre',g)
+        
+                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+                if m:
+                    for g in m.group(1).split(','):
+                        self.story.addToList('characters',g)
+        
+                m = re.match(r".*?Warnings: (.+).*?",metastr)
+                if m:
+                    for w in m.group(1).split(','):
+                        if w != 'Now Warnings':
+                            self.story.addToList('warnings',w)
+        
+                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
+                if m:
+                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
 
-            # Updated can have more than one space after it. <shrug>
-            m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-            if m:
-                self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
+                # Updated can have more than one space after it. <shrug>
+                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
+                if m:
+                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
 
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         ## most adapters use BeautifulStoneSoup here, but non-Stone
         ## allows nested div tags.

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -1,172 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return HennethAnnunNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class HennethAnnunNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','htan')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.henneth-annun.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
-
-    def getSiteURLPattern(self):
-        return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-            
-        if "We're sorry. This story is not available." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: This story is not available.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('h2', {'id':'page_heading'})
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
-        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        self.story.setMetadata('numWords', soup.find('tr', {'class':'foot'}).findAll('td')[1].text)
-
-        self.setDescription(url,soup.find('div', {'id':'summary'}))
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        info = soup.find('div', {'id':'storyinformation'})
-        labels=info.findAll('b')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-                
-            if 'Completion' in label:
-                if 'Complete' in value.string:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.string)
-
-            if 'Era:' in label:
-                self.story.addToList('category',value.string)
-
-            if 'Genre' in label:
-                self.story.addToList('genre',value.string)
-                
-        labels=info.findAll('strong')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-            
-            if 'Author' in label:
-                value=value.nextSibling
-                self.story.setMetadata('authorId',value['href'].split('=')[1])
-                self.story.setMetadata('authorUrl','http://'+self.host+'/'+value['href'])
-                self.story.setMetadata('author',value.string)
-                
-            if 'Post' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated:' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                
-        for char in soup.findAll('a', href=re.compile(r"/resources/bios_view.cfm\?scid=\d+")):
-            self.story.addToList('characters',stripHTML(char))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'class' : 'block chapter'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -1,232 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return HLFictionNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class HLFictionNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','hlf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'hlfiction.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select')
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                # just in case there's tags, like <i> in chapter titles. 
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                break
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
-        for labelspan in labels:
-            label = labelspan.string
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'classification':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
-
-            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -53,7 +51,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
@@ -62,6 +60,11 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','hpfdm') # XXX
 
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d" # XXX
@@ -71,9 +74,8 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'www.hpfandom.net' # XXX
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/eff/viewstory.php?sid=1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=")+r"\d+$"
@@ -82,7 +84,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -183,22 +185,21 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
             value = td.nextSibling.string
             #print("\nlabel:%s\nvalue:%s\n"%(label,value))
 
-            if 'Category' in label and value:
+            if 'Category' in label:
                 cats = td.parent.findAll('a',href=re.compile(r'categories.php'))
                 catstext = [cat.string for cat in cats]
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
-            if 'Characters' in label and value: # this site can have Character label with no
-                                                # values, apparently.  Others as a precaution.
+            if 'Characters' in label:
                 for char in value.split(','):
                     self.story.addToList('characters',char.strip())
 
-            if 'Genre' in label and value:
+            if 'Genre' in label:
                 for genre in value.split(','):
                     self.story.addToList('genre',genre.strip())
 
-            if 'Warnings' in label and value:
+            if 'Warnings' in label:
                 for warning in value.split(','):
                     if warning.strip() != 'none':
                         self.story.addToList('warnings',warning.strip())
@@ -212,7 +213,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # There's no good wrapper around the chapter text. :-/

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -1,224 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return HPFanficArchiveComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class HPFanficArchiveComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','hpffa')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.hpfanficarchive.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/stories/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/stories/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            val = labelspan.nextSibling
-            value = unicode('')
-            while val and not defaultGetattr(val,'class') == 'label':
-                value += unicode(val)
-                val = val.nextSibling
-            label = labelspan.string
-            #print("label:%s\nvalue:%s"%(label,value))
-
-            if 'Summary' in label:
-                self.setDescription(url,value)
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', stripHTML(value))
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', stripHTML(value))
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
-                for ship in ships:
-                    self.story.addToList('ships',ship.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in stripHTML(value):
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/stories/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-        
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -1,283 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return IkEternalNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class IkEternalNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ike')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.ik-eternal.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=1"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data,selfClosingTags=('p')) #poor formatting of the paragraphs in the title page
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        asoup = soup.find('div', {'class': 'listbox'})
-        for a in asoup.findAll('p'):
-            a.name='br'
-        labels = asoup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -1,290 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ImagineEFicComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ImagineEFicComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ime')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y.%m.%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'imagine.e-fic.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_indeathnet.py

@@ -1,200 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return InDeathNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class InDeathNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-            
-            # normalized story URL.
-            self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/')
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','idn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'indeath.net'
-
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/blog/archive/123-story-in-death/"
-
-    def getSiteURLPattern(self):
-        # http://www.indeath.net/blog/archive/169-ransom-in-death/
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"/blog/(archive/)?(?P<id>\d+)\-(?P<name>[a-z0-9\-]*)/?$"     
-        
-            
-    def getDateFromComponents(self, postmonth, postday):
-        ym = re.search("Entries\ in\ (?P<mon>January|February|March|April|May|June|July|August|September|October|November|December)\ (?P<year>\d{4})",postmonth)
-        d = re.search("(?P<day>\d{2})\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)",postday)
-        postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
-        return postdate
-
-    def getAuthorData(self):
-        
-        mainUrl = self.url.replace("/archive","")
-        
-        try:
-            maindata = self._fetchUrl(mainUrl)
-            
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
-            else:
-                raise e
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        mainsoup = bs.BeautifulSoup(maindata)
-
-        # find first entry
-        e = mainsoup.find('div',{'class':"entry"})
-        
-        # get post author as author
-        d = e.find('div',{'class':"desc"})
-        a = d.find('strong')
-        self.story.setMetadata('author',a.contents[0].string.strip())
-        
-        # Don't seem to be able to get author pages anymore
-        self.story.setMetadata('authorUrl','http://www.indeath.net/')
-        self.story.setMetadata('authorId','0')
-        
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        try:
-            data = self._fetchUrl(url)
-            
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
-            else:
-                raise e
-                
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        h = soup.find('a', id="blog_title")
-        t = h.find('span')
-        self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
-        
-        s = t.find('div')        
-        if s != None:
-            self.setDescription(url,s)
-
-        # Get Author from main blog page since it's not reliably on the archive page
-        self.getAuthorData()
-
-        # Find the chapters:
-        chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
-
-        #reverse the list since newest at the top
-        chapters.reverse()
-        
-        # Get date published & updated from first & last entries
-        posttable=soup.find('div', id="main_column")
-        
-        postmonths=posttable.findAll('th', text=re.compile(r'Entries\ in\ '))
-        postmonths.reverse()
-        
-        postdates=posttable.findAll('span', _class="desc", text=re.compile('\d{2}\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'))
-        postdates.reverse()
-        
-        self.story.setMetadata('datePublished',self.getDateFromComponents(postmonths[0],postdates[0]))
-        self.story.setMetadata('dateUpdated',self.getDateFromComponents(postmonths[len(postmonths)-1],postdates[len(postdates)-1]))
-        
-        # Process List of Chapters              
-        self.story.setMetadata('numChapters',len(chapters))
-        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
-            else:
-                ct = stripHTML(chapter)
-                tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
-                if tnew:
-                    chaptertitle = tnew.group('newtitle')
-                else:
-                    chaptertitle = ct
-                self.chapterUrls.append((chaptertitle,chapter['href']))
-
-        
-        
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        
-        #chapter=bs.BeautifulSoup('<div class="story"></div>')
-        data = self._fetchUrl(url)
-        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))
-
-        chapter = soup.find("div", "entry_content")
-        
-        if None == chapter:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,chapter)
-

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -53,7 +51,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -62,6 +60,11 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ksa') # XXX
 
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Star Trek") # XXX
+
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b/%d/%Y" # XXX
@@ -75,9 +78,8 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'ksarchive.com' # XXX
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -102,7 +104,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -135,7 +137,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
+                logging.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -165,7 +167,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',stripHTML(a))
+        self.story.setMetadata('author',a.string)
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -188,20 +190,14 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
-            label = stripHTML(labelspan)
+            label = labelspan.string
 
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
-                    # poor HTML(unclosed <p> for one) can cause run on
-                    # over the next label.
-                    if '<span class="label">' in svalue:
-                        svalue = svalue[0:svalue.find('<span class="label">')]
-                        break
-                    else:
-                        value = value.nextSibling
+                    value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -213,50 +209,48 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [stripHTML(cat) for cat in cats]
+                catstext = [cat.string for cat in cats]
                 for cat in catstext:
-                    # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
-                    # tag in the desc once.
-                    if cat and cat.strip() in ('Poetry','Essays'):
-                        self.story.addToList('category',stripHTML(cat))
+                    if cat.string.strip() in ('Poetry','Essays'):
+                        self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
                 self.story.addToList('characters','Kirk')
                 self.story.addToList('characters','Spock')
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [stripHTML(char) for char in chars]
+                charstext = [char.string for char in chars]
                 for char in charstext:
-                    self.story.addToList('characters',stripHTML(char))
+                    self.story.addToList('characters',char.string)
 
             ## Not all sites use Genre, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [stripHTML(genre) for genre in genres]
+                genrestext = [genre.string for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
-                    self.story.addToList('genre',stripHTML(genre))
+                    self.story.addToList('genre',genre.string)
 
             ## In addition to Genre (which is very site specific) KSA
             ## has 'Story Type', which is much more what most sites
             ## call genre.
             if 'Story Type' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
-                genrestext = [stripHTML(genre) for genre in genres]
+                genrestext = [genre.string for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
-                    self.story.addToList('genre',stripHTML(genre))
+                    self.story.addToList('genre',genre.string)
 
             ## Not all sites use Warnings, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Warnings' in label:
                 warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [stripHTML(warning) for warning in warnings]
+                warningstext = [warning.string for warning in warnings]
                 self.warning = ', '.join(warningstext)
                 for warning in warningstext:
-                    self.story.addToList('warnings',stripHTML(warning))
+                    self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
                 if 'Yes' in value:
@@ -275,21 +269,18 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = stripHTML(a)
+            series_name = a.string
             series_url = 'http://'+self.host+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
             
         except:
             # I find it hard to care if the series parsing fails
@@ -298,7 +289,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -1,251 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return LibraryOfMoriaComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class LibraryOfMoriaComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','lom')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.libraryofmoria.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/a/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/a/viewstory.php?sid=")+r"\d+$"
-
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            addurl = "&ageconsent=ok&warning=3"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/a/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/a/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Type' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warning' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/a/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_literotica.py

@@ -1,267 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import urlparse
-import time
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter, makeDate
-
-class LiteroticaSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-
-        self.story.setMetadata('siteabbrev','litero')
-
-        # normalize to first chapter.  Not sure if they ever have more than 2 digits.
-        storyId = self.parsedUrl.path.split('/',)[2]
-        # replace later chapters with first chapter but don't remove numbers
-        # from the URL that disambiguate stories with the same title.
-        storyId = re.sub("-ch-?\d\d", "", storyId)
-        self.story.setMetadata('storyId', storyId)
-
-        ## accept m(mobile)url, but use www.
-        url = re.sub("^(www|german|spanish|french|dutch|italian|romanian|portuguese|other)\.i",
-                              "\1",
-                              url)
-
-        ## strip ?page=...
-        url = re.sub("\?page=.*$", "", url)
-
-        ## set url
-        self._setURL(url)
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = '%m/%d/%y'
-
-    @staticmethod
-    def getSiteDomain():
-        return 'literotica.com'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.literotica.com',
-                'www.i.literotica.com',
-                'german.literotica.com',
-                'german.i.literotica.com',
-                'spanish.literotica.com',
-                'spanish.i.literotica.com',
-                'french.literotica.com',
-                'french.i.literotica.com',
-                'dutch.literotica.com',
-                'dutch.i.literotica.com',
-                'italian.literotica.com',
-                'italian.i.literotica.com',
-                'romanian.literotica.com',
-                'romanian.i.literotica.com',
-                'portuguese.literotica.com',
-                'portuguese.i.literotica.com',
-                'other.literotica.com',
-                'other.i.literotica.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title"
-
-    def getSiteURLPattern(self):
-        return r"https?://(www|german|spanish|french|dutch|italian|romanian|portuguese|other)(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
-
-    def extractChapterUrlsAndMetadata(self):
-        """
-        NOTE: Some stories can have versions, 
-              e.g. /my-story-ch-05-version-10
-        NOTE: If two stories share the same title, a running index is added,
-              e.g.: /my-story-ch-02-1
-        Strategy:
-            * Go to author's page, search for the current story link,
-            * If it's in a tr.root-story => One-part story
-                * , get metadata and be done
-            * If it's in a tr.sl => Chapter in series
-                * Search up from there until we find a tr.ser-ttl (this is the
-                story)
-                * Gather metadata
-                * Search down from there for all tr.sl until the next
-                tr.ser-ttl, foreach
-                    * Chapter link is there
-        """
-
-        if not (self.is_adult or self.getConfig("is_adult")):
-            raise exceptions.AdultCheckRequired(self.url)
-
-        logger.debug("Chapter/Story URL: <%s> " % self.url)
-        try:
-            data1 = self._fetchUrl(self.url)
-            soup1 = bs.BeautifulSoup(data1)
-            #strip comments from soup
-            [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # author
-        a = soup1.find("span", "b-story-user-y")
-        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'][0])
-        authorurl = a.a['href']
-        if authorurl.startswith('//'):
-            authorurl = self.parsedUrl.scheme+':'+authorurl
-        self.story.setMetadata('authorUrl', authorurl)
-        self.story.setMetadata('author', a.text)
-
-        # get the author page
-        try:
-            dataAuth = self._fetchUrl(authorurl)
-            soupAuth = bs.BeautifulSoup(dataAuth)
-            #strip comments from soup
-            [comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, bs.Comment))]
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(authorurl)
-            else:
-                raise e
-
-        ## Find link to url in author's page
-        ## site has started using //domain.name/asdf urls remove https?: from front
-        storyLink = soupAuth.find('a', href=self.url[self.url.index(':')+1:])
-
-        if storyLink is not None:
-            urlTr = storyLink.parent.parent
-            if urlTr['class'] == "sl":
-                isSingleStory = False
-            else:
-                isSingleStory = True
-        else:
-            raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (url, authorurl))
-
-        if isSingleStory:
-            self.story.setMetadata('title', storyLink.text)
-            self.story.setMetadata('description', urlTr.findAll("td")[1].text)
-            self.story.addToList('eroticatags', urlTr.findAll("td")[2].text)
-            date = urlTr.findAll('td')[-1].text
-            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
-            self.chapterUrls = [(storyLink.text, self.url)]
-        else:
-            seriesTr = urlTr.previousSibling
-            while seriesTr['class'] != 'ser-ttl':
-                seriesTr = seriesTr.previousSibling
-            m = re.match("^(?P<title>.*?):\s(?P<numChapters>\d+)\sPart\sSeries$", seriesTr.find("strong").text)
-            self.story.setMetadata('title', m.group('title'))
-
-            ## Walk the chapters
-            chapterTr = seriesTr.nextSibling
-            self.chapterUrls = []
-            dates = []
-            descriptions = []
-            while chapterTr is not None and chapterTr['class'] == 'sl':
-                descriptions.append(chapterTr.findAll("td")[1].text)
-                chapterLink = chapterTr.find("td", "fc").find("a")
-                self.chapterUrls.append((chapterLink.text, "http:" + chapterLink["href"]))
-                self.story.addToList('eroticatags', chapterTr.findAll("td")[2].text)
-                dates.append(makeDate(chapterTr.findAll('td')[-1].text, self.dateformat))
-                chapterTr = chapterTr.nextSibling
-
-            ## Set description to joint chapter descriptions
-            self.story.setMetadata('description', " / ".join(descriptions))
-
-            ## Set the oldest date as publication date, the newest as update date
-            dates.sort()
-            self.story.setMetadata('datePublished', dates[0])
-            self.story.setMetadata('dateUpdated', dates[-1])
-
-            # normalize on first chapter URL.
-            self._setURL(self.chapterUrls[0][1])
-
-        self.story.setMetadata('numChapters', len(self.chapterUrls))
-                
-        # set storyId to 'title-author' to avoid duplicates
-        # self.story.setMetadata('storyId', 
-        #     re.sub("[^a-z0-9]", "", self.story.getMetadata('title').lower())
-        #     + "-"
-        #     + re.sub("[^a-z0-9]", "", self.story.getMetadata('author').lower()))
-
-        return
-
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from <%s>' % url)
-        data1 = self._fetchUrl(url)
-        # brute force approach to replace the wrapping <p> tag.  If
-        # done by changing tag name, it causes problems with nested
-        # <p> tags.
-        data1 = data1.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
-        soup1 = bs.BeautifulSoup(data1)
-
-        #strip comments from soup
-        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
-
-        # get story text
-        story1 = soup1.find('div', 'b-story-body-x').div
-        #print("story1:%s"%story1)
-        # story1.name='div'
-        story1.append('<br />')
-        storytext = self.utf8FromSoup(url,story1)
-
-        # find num pages
-        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
-        logger.debug("pages: "+str(pgs))
-
-        # get all the pages
-        for i in xrange(2, pgs+1):
-            try:
-                logger.debug("fetching page "+str(i))
-                time.sleep(0.5)
-                data2 = self._fetchUrl(url, {'page': i})
-                # brute force approach to replace the wrapping <p> tag.  If
-                # done by changing tag name, it causes problems with nested
-                # <p> tags.
-                data2 = data2.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
-                soup2 = bs.BeautifulSoup(data2)
-                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
-                story2 = soup2.find('div', 'b-story-body-x').div
-                # story2.name='div'
-                story2.append('<br />')
-                storytext += self.utf8FromSoup(url,story2)
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(url)
-                else:
-                    raise e
-        return storytext
-
-
-def getClass():
-    return LiteroticaSiteAdapter
-
-
-

fanficdownloader/adapters/adapter_lotrfanfictioncom.py

@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2014 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from base_efiction_adapter import BaseEfictionAdapter
-
-class TheLOTRFanFictionSiteAdapter(BaseEfictionAdapter):
-
-    @staticmethod
-    def getSiteDomain():
-        return 'lotrfanfiction.com'
-
-    @classmethod
-    def getSiteAbbrev(seluuf):
-        return 'lotrff'
-
-    @classmethod
-    def getDateFormat(self):
-        return "%d/%m/%y"
-
-def getClass():
-    return TheLOTRFanFictionSiteAdapter

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -1,238 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return LumosSycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class LumosSycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','lsph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'lumos.sycophanthex.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=19"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Age Consent Required" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        rating=pt.text.split('(')[1].split(')')[0]
-        self.story.setMetadata('rating', rating)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-				
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-		
-        labels = soup.findAll('span',{'class':'label'})
-		
-        value = labels[0].previousSibling
-        svalue = ""
-        while value != None:
-            val = value
-            value = value.previousSibling
-        while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-		
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value.split(' -')[0])
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Complete' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -17,7 +17,6 @@
 
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -43,7 +42,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
         else:
@@ -55,9 +54,8 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.mediaminer.org'
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+cls.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+self.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
 
     def getSiteURLPattern(self):
         ##  http://www.mediaminer.org/fanfic/view_st.php/76882
@@ -68,7 +66,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -107,10 +105,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
         if title.has_key('colspan'):
-            titlet = stripHTML(title)
+            titlet = title.text
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+            titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
         ## The story title is difficult to reliably parse from the
         ## story pages.  Getting it from the author page is, but costs
@@ -190,7 +188,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         data=self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,
@@ -216,7 +214,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
             return self.utf8FromSoup(url,anchor)
         
         else:
-            logger.debug('Using kludgey text find for older mediaminer story.')
+            logging.debug('Using kludgey text find for older mediaminer story.')
             ## Some older mediaminer stories are unparsable with BeautifulSoup.
             ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
             ## Story stuff falls between:

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -1,294 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return MerlinFicDtwinsCoUk
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class MerlinFicDtwinsCoUk(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','mrfd')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'merlinfic.dtwins.co.uk'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-                    
-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for ship in ships:
-                    self.story.addToList('ships',ship.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -53,7 +51,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -62,6 +60,11 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','mw') # XXX
 
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Queer as Folk") # XXX
+
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y" # XXX
@@ -71,9 +74,8 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'www.midnightwhispers.ca' # XXX
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -98,7 +100,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -131,7 +133,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
+                logging.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -258,13 +260,10 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
             storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
             
         except:
             # I find it hard to care if the series parsing fails
@@ -273,7 +272,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -53,7 +51,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -61,25 +59,25 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','mgln') # XXX
 
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y" # XXX
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
-        # The site domain.
-        return 'fanfiction.mugglenet.com'
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfiction.mugglenet.com' # XXX
 
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -103,13 +101,13 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId')
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -132,7 +130,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -171,7 +169,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
+                logging.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -194,7 +192,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
+        self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -305,16 +303,13 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
             
         except:
             # I find it hard to care if the series parsing fails
@@ -323,7 +318,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -1,213 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NationalLibraryNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NationalLibraryNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only storyid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ntlb')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m-%d-%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        return 'national-library.net'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.national-library.net','national-library.net']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        # ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: 
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?storyid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('h1')
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for p in soup.findAll('p'):
-            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
-            if len(chapters) > 0:
-                for chapter in chapters:
-                    # just in case there's tags, like <i> in chapter titles.
-                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-                break
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        self.story.setMetadata('status', 'Completed')
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('b')
-        for x in range(2,len(labels)):
-            value = labels[x].nextSibling
-            label = labels[x].string
-            
-            if 'Summary' in label:
-                self.setDescription(url,value)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', stripHTML(value.nextSibling))
-
-            if 'Word Count' in label:
-                self.story.setMetadata('numWords', value.string)
-
-            if 'Category' in label:
-                for cat in value.string.split(', '):
-                    self.story.addToList('category',cat)
-            if 'Crossover Shows' in label:
-                for cat in value.string.split(', '):
-                    if "No Show" not in cat:
-                        self.story.addToList('category',cat)
-
-            if 'Character' in label:
-                for char in value.string.split(', '):
-                    self.story.addToList('characters',char)
-                    
-            if 'Pairing' in label:
-                for char in value.string.split(', '):
-                    self.story.addToList('ships',char)
-                    
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    self.story.addToList('warnings',warning)
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Series' in label:
-                self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
-                self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
-                
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
-            
-        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
-        for genre in a:
-            self.story.setMetadata('genre', genre)
-        
-        a=story.findNext(text=re.compile('Archived'))
-        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div')
-        
-        # bit messy since higly inconsistent
-        for p in soup.findAll('p', {'align' : 'center'}):
-            p.extract()
-        p = soup.findAll('p')
-        for x in range(0,3):
-            p[x].extract()
-        if "Chapters: " in stripHTML(p[3]):
-            p[3].extract()
-        for x in range(len(p)-2,len(p)-1):
-            p[x].extract()
-
-        for p in soup.findAll('h1'):
-            p.extract()
-        for p in soup.findAll('h3'):
-            p.extract()
-        for p in soup.findAll('a'):
-            p.extract()
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -1,219 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NCISFicComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NCISFicComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only storyid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ncisf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m-%d-%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        return 'ncisfic.com'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.ncisfic.com','ncisfic.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?storyid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('h1')
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for p in soup.findAll('p'):
-            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
-            if len(chapters) > 0:
-                for chapter in chapters:
-                    # just in case there's tags, like <i> in chapter titles.
-                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-                break
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        self.story.setMetadata('status', 'Completed')
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('b')
-        for x in range(2,len(labels)):
-            value = labels[x].nextSibling
-            label = labels[x].string
-            
-            if 'Summary' in label:
-                self.setDescription(url,value)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', stripHTML(value.nextSibling))
-
-            if 'Word Count' in label:
-                self.story.setMetadata('numWords', value.string)
-
-            if 'Category' in label:
-                for cat in value.string.split(', '):
-                    self.story.addToList('category',cat)
-            if 'Crossover Shows' in label:
-                for cat in value.string.split(', '):
-                    if "No Show" not in cat:
-                        self.story.addToList('category',cat)
-
-            if 'Character' in label:
-                for char in value.string.split(', '):
-                    self.story.addToList('characters',char)
-                    
-            if 'Pairing' in label:
-                for char in value.string.split(', '):
-                    self.story.addToList('ships',char)
-                    
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    self.story.addToList('warnings',warning)
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Series' in label:
-                if "No Series" not in value.nextSibling.string:
-                    self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
-                    self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
-                
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
-        
-        a=story.findNext('font')
-        if 'Complete' in a.string:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-            
-        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
-        for genre in a:
-            self.story.setMetadata('genre', genre)
-        
-        a=story.findNext(text=re.compile('Archived'))
-        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div')
-        
-        # bit messy since higly inconsistent
-        for p in soup.findAll('p', {'align' : 'center'}):
-            p.extract()
-        p = soup.findAll('p')
-        for x in range(0,3):
-            p[x].extract()
-        if "Chapters: " in stripHTML(p[3]):
-            p[3].extract()
-        for x in range(len(p)-2,len(p)-1):
-            p[x].extract()
-
-        for p in soup.findAll('h1'):
-            p.extract()
-        for p in soup.findAll('h3'):
-            p.extract()
-        for p in soup.findAll('a'):
-            p.extract()
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -1,210 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NCISFictionNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NCISFictionNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["iso-8859-1",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
-                         +"/chapters.php?stid="+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ncisfn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.ncisfiction.net'
-
-    ## Changed from www.ncisfiction.com to www.ncisfiction.net Oct
-    ## 2012 due to the ncisfiction.com domain expiring.  Still accept
-    ## .com domains for existing updates, etc.
-    
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.ncisfiction.net','www.ncisfiction.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/story.php?stid=01234 http://"+cls.getSiteDomain()+"/chapters.php?stid=1234"
-
-    def getSiteURLPattern(self):
-        return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+'
-       
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulStoneSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'class' : 'main_title'})
-        
-        aut = a.find('a')
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        
-        aut.extract()
-        self.story.setMetadata('title',stripHTML(a)[:len(stripHTML(a))-2])
-
-        # Find the chapters:
-        i=0
-        chapters=soup.findAll('table', {'class' : 'story_table'})
-        for chapter in chapters:
-            ch=chapter.find('a')
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/'+ch['href']))
-            if i == 0:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
-            if i == len(chapters)-1:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
-            i=i+1
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        info = soup.find('table', {'class' : 'story_info'})
-        
-        # no convenient way to calculate word count as it is logged differently for stories with and without series
-        
-        labels = info.findAll('tr')
-        for tr in labels:
-            value = tr.find('td')
-            label = tr.find('th').string
-
-            if 'Summary' in label:
-                self.setDescription(url,value)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.string)
-
-            if 'Category' in label:
-                cats = value.findAll('a')
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = value.findAll('a')
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Pairing' in label:
-                ships = value.findAll('a')
-                for ship in ships:
-                    self.story.addToList('ships',ship.string)
-
-            if 'Genre' in label:
-                genres = value.findAll('a')
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = value.findAll('a')
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Status' in label:
-                if 'not completed' in value.text:
-                    self.story.setMetadata('status', 'In-Progress')
-                else:
-                    self.story.setMetadata('status', 'Completed')
-                    
-        try:
-            # Find Series name from series URL.
-            a = soup.find('div',{'class' : 'sub_header'})
-            series_name = a.find('a').string
-            i = a.text.split('#')[1]
-            self.setSeries(series_name, i)
-            self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href'])
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'class' : 'story_text'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_netraptororg.py

@@ -1,213 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NetRaptorOrgAdapter
-
-class NetRaptorOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','netrap')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'netraptor.org'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pagetitle = soup.find('div',{'id':'pagetitle'})
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-				
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -53,7 +51,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -62,6 +60,11 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','nfa') # XXX
 
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","NCIS") # XXX
+
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y" # XXX
@@ -75,9 +78,8 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'nfacommunity.com' # XXX
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -102,7 +104,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -135,7 +137,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
+                logging.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -157,7 +159,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
+        self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -259,16 +261,13 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
             
         except:
             # I find it hard to care if the series parsing fails
@@ -277,7 +276,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -1,237 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NHAMagicalWorldsUsAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nha')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'nha.magical-worlds.us'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a')
-            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
-                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',stripHTML(a))
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'viewuser.php'))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'br':
-            svalue += unicode(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-        
-        #does not provide convenient way to get word count
-        labels = info.findAll('i')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.split(' -')[0])
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if 'None' not in genre:
-                        self.story.addToList('genre',genre.split(' -')[0])
-            
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if 'None' not in char:
-                        self.story.addToList('characters',char.split(' -')[0])
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if 'None' not in warning:
-                        self.story.addToList('warnings',warning.split(' -')[0])
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"id" : "story"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -1,177 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NickAndGregNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NickAndGregNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nag')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y/%m/%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.nickandgreg.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
-    
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&i=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        chapters = soup.find('select')
-        for chapter in chapters.findAll('option'):
-            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
-            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                break
-        
-        self.setDescription(url,div.find('br').nextSibling)
-        
-        a=div.text.split('Rating:')
-        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
-        
-        a=div.text.split('Characters:')
-        if len(a) == 2: 
-        	for char in a[1].split(' -')[0].split(', '):
-        	    self.story.addToList('characters',char)
-        	    
-        a=div.text.split('Genres:')
-        if len(a) == 2: 
-        	for genre in a[1].split(' -')[0].split(', '):
-        	    self.story.addToList('genre',genre)
-        	    
-        a=div.text.split('Warnings:')
-        if len(a) == 2: 
-        	for warn in a[1].split(' -')[0].split(', '):
-        	    if 'none' not in warn:
-        	        self.story.addToList('warnings',warn)
-        	    
-        a=div.text.split('Completed:')
-        if len(a) ==2:
-            if 'Yes' in a[1]:
-                self.story.setMetadata('status', 'Completed')
-            else:
-                self.story.setMetadata('status', 'In-Progress')
-                
-        a=div.text.split('Published:')
-        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
-        
-        a=div.text.split('Updated:')
-        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        # wrap a div around it.
-        divsoup = bs.BeautifulStoneSoup('<div class="story"></div>',
-                                        selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        div = divsoup.find('div')
-        div.append(soup.find('table', {'class' : 'tblborder6'}))
-
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nocturnallightnet.py

@@ -1,177 +0,0 @@
-import re
-import urllib2
-import urlparse
-
-from .. import BeautifulSoup
-
-from base_adapter import BaseSiteAdapter, makeDate
-from .. import exceptions
-
-
-def getClass():
-    return NocturnalLightNetAdapter
-
-
-# yields Tag _and_ NavigableString siblings from the given tag. The
-# BeautifulSoup findNextSiblings() method for some reasons only returns either
-# NavigableStrings _or_ Tag objects, not both.
-def _yield_next_siblings(tag):
-    sibling = tag.nextSibling
-    while sibling:
-        yield sibling
-        sibling = sibling.nextSibling
-
-
-class NocturnalLightNetAdapter(BaseSiteAdapter):
-    SITE_ABBREVIATION = 'nln'
-    SITE_DOMAIN = 'nocturnal-light.net'
-    BASE_URL = 'http://' + SITE_DOMAIN + '/fanfiction/'
-    STORY_URL_TEMPLATE = BASE_URL + 'story/%s'
-    AUTHORS_URL_TEMPLATE = BASE_URL + 'authors/%s'
-
-    DATETIME_FORMAT = '%m-%d-%y'
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        url_tokens = self.parsedUrl.path.split('/')
-        story_id = url_tokens[url_tokens.index('story') + 1]
-
-        self.story.setMetadata('storyId', story_id)
-        self._setURL(self.STORY_URL_TEMPLATE % story_id)
-        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
-
-    def _customized_fetch_url(self, url, exception=None, parameters=None):
-        if exception:
-            try:
-                data = self._fetchUrl(url, parameters)
-            except urllib2.HTTPError:
-                raise exception(self.url)
-        # Just let self._fetchUrl throw the exception, don't catch and
-        # customize it.
-        else:
-            data = self._fetchUrl(url, parameters)
-
-        return BeautifulSoup.BeautifulSoup(data)
-
-    @staticmethod
-    def getSiteDomain():
-        return NocturnalLightNetAdapter.SITE_DOMAIN
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return cls.STORY_URL_TEMPLATE % 1234
-
-    def getSiteURLPattern(self):
-        return re.escape(self.STORY_URL_TEMPLATE[:-2]) + r'\d+.*$'
-
-    def extractChapterUrlsAndMetadata(self):
-        soup = self._customized_fetch_url(self.url)
-
-        # Since no 404 error code we have to raise the exception ourselves.
-        # A title that is just 'by' indicates that there is no author name
-        # and no story title available.
-        if soup.title.string.strip() == 'by':
-            raise exceptions.StoryDoesNotExist(self.url)
-
-        # "storycontent" is found in a single-chapter story
-        author_anchor = soup.find('div', id=lambda id: id in ('main', 'storycontent')).h1.a
-        self.story.setMetadata('author', author_anchor.string)
-
-        url_tokens = author_anchor['href'].split('/')
-        author_id = url_tokens[url_tokens.index('authors')+1]
-        self.story.setMetadata('authorId', author_id)
-        self.story.setMetadata('authorUrl', self.AUTHORS_URL_TEMPLATE % author_id)
-
-        chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/'))
-        for chapter_anchor in chapter_anchors:
-            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
-            self.chapterUrls.append((chapter_anchor.string, url))
-
-        author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
-        soup = self._customized_fetch_url(author_url)
-        story_id = self.story.getMetadata('storyId')
-        for listbox in soup('div', {'class': 'listbox'}):
-            url_tokens = listbox.a['href'].split('/')
-            # Found the div containing the story's metadata; break the loop and
-            # parse the element
-            if story_id == url_tokens[url_tokens.index('story')+1]:
-                break
-        else:
-            raise exceptions.FailedToDownload(self.url)
-
-        title = listbox.a.string
-        self.story.setMetadata('title', title)
-
-        # No chapter anchors found in the original story URL, so the story has
-        # only a single chapter.
-        if not chapter_anchors:
-            self.chapterUrls.append((title, self.url))
-
-        for b_tag in listbox('b'):
-            key = b_tag.string.strip(':')
-            try:
-                value = b_tag.nextSibling.string.replace('•', '').strip(': ')
-            # This can happen with some fancy markup in the summary. Just
-            # ignore this error and set value to None, the summary parsing
-            # takes care of this
-            except AttributeError:
-                value = None
-
-            if key == 'Summary':
-                contents = []
-                keep_summary_html = self.getConfig('keep_summary_html')
-
-                for sibling in _yield_next_siblings(b_tag):
-                    if isinstance(sibling, BeautifulSoup.Tag):
-                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
-                            break
-
-                        if keep_summary_html:
-                            contents.append(self.utf8FromSoup(author_url, sibling))
-                        else:
-                            contents.append(''.join(sibling(text=True)))
-                    else:
-                        contents.append(sibling)
-
-                # Pop last break line tag
-                contents.pop()
-                self.story.setMetadata('description', ''.join(contents))
-
-            elif key == 'Category':
-                for sibling in b_tag.findNextSiblings(['a', 'b']):
-                    if sibling.name == 'b':
-                        break
-
-                    self.story.addToList('category', sibling.string)
-
-            elif key == 'Rating':
-                self.story.setMetadata('rating', value)
-
-            elif key == 'Chapters':
-                self.story.setMetadata('numChapters', int(value))
-
-                # Also parse reviews number which lies right after the chapters
-                # section
-                reviews_anchor = b_tag.findNextSibling('a')
-                reviews = reviews_anchor.string.split(' ')[1].strip('()')
-                self.story.setMetadata('reviews', reviews)
-
-            elif key == 'Completed':
-                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
-
-            elif key == 'Date Added':
-                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
-
-            elif key == 'Last Updated':
-                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
-
-            elif key == 'Read':
-                self.story.setMetadata('readings', value.split()[0])
-
-        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
-            raise exceptions.AdultCheckRequired(self.url)
-
-    def getChapterText(self, url):
-        soup = self._customized_fetch_url(url)
-        return self.utf8FromSoup(url, soup.find('div', id='storytext'))

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -1,263 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return OcclumencySycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','osph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'occlumency.sycophanthex.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'This story contains adult content and/or themes.' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Logout" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'class' : 'border'}):
-            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                self.story.setMetadata('title',stripHTML(a))
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-			
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'table':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = info.findAll('b')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word Count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if genre != 'none':
-                        self.story.addToList('genre',genre)
-						
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if char != 'none':
-                        self.story.addToList('characters',char)
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if warning != ' none':
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-        data = data.replace('<div align="left"', '<div align="left">')
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr'))
-        
-        story = soup.find('div', {"align" : "left"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -1,270 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return OneDirectionFanfictionComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','odf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'onedirectionfanfiction.com'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']
-
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-        
-        ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-    
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-        
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Age Consent Required" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while value and not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -1,240 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2, urllib, cookielib
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PhoenixSongNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PhoenixSongNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','phs')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.phoenixsong.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfiction/story/1234/"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/story/")+r"\d+/?$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Please login to continue.' in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-		
-        if self.password:
-            params['txtusername'] = self.username
-            params['txtpassword'] = self.password
-        else:
-            params['txtusername'] = self.getConfig("username")
-            params['txtpassword'] = self.getConfig("password")
-        #params['remember'] = '1'
-        params['login'] = 'Login'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/users/processlogin.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['txtusername']))
-        d = self._fetchUrl(loginUrl, params)
-    
-        if 'Please login to continue.' in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['txtusername']))
-            raise exceptions.FailedToLogin(url,params['txtusername'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            if self.getConfig('force_login'):
-                self.performLogin(url)
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        b = soup.find('div', {'id' : 'nav25'})
-        a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
-        a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        chapters = soup.find('select')
-        if chapters == None:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-            for b in soup.findAll('b'):
-                if b.text == "Updated":
-                    date = b.nextSibling.string.split(': ')[1].split(',')
-                    self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
-                    self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
-        else:
-            i = 0
-            chapters = chapters.findAll('option')
-            for chapter in chapters:
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['value']))
-                if i == 0:
-                    self.story.setMetadata('storyId',chapter['value'].split('/')[3])
-                    head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b')
-                    for b in head:
-                        if b.text == "Updated":
-                            date = b.nextSibling.string.split(': ')[1].split(',')
-                            self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
-                            
-                if  i == (len(chapters)-1):
-                    head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b')
-                    for b in head:
-                        if b.text == "Updated":
-                            date = b.nextSibling.string.split(': ')[1].split(',')
-                            self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
-                i = i+1
-				
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))        	
-				
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        info = asoup.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        while info != None:
-            info = info.findNext('div')
-            b = info.find('b')
-            val = b.nextSibling
-            
-            if 'Rating' in b.string:
-                self.story.setMetadata('rating', val.string.split(': ')[1])
-				
-            if 'Words' in b.string:
-                self.story.setMetadata('numWords', val.string.split(': ')[1])
-				
-            if 'Setting' in b.string:
-                self.story.addToList('category', val.string.split(': ')[1])
-				
-            if 'Status' in b.string:
-                if 'Completed' in val:
-                    val = 'Completed'
-                else:
-                    val = 'In-Progress'
-                self.story.setMetadata('status', val)
-            
-            if 'Summary' in b.string:
-                b.extract()
-                info.find('br').extract()
-                self.setDescription(url,info)
-                break
-	
-	
-	# grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
-        for p in soup.findAll('p'):
-            if "This is for problems with the formatting or the layout of the chapter." in stripHTML(p):
-                break
-            chapter.append(p)
-			
-        for a in chapter.findAll('div'):
-            a.extract()
-        for a in chapter.findAll('table'):
-            a.extract()
-        for a in chapter.findAll('script'):
-            a.extract()
-        for a in chapter.findAll('form'):
-            a.extract()
-        for a in chapter.findAll('textarea'):
-            a.extract()
-
-
-        if None == chapter:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,chapter)

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -1,301 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PommeDeSangComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PommeDeSangComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # pommedesang.com has two 'sections', shown in URL as
-        # 'efiction' and 'sds' that change how things should be
-        # handled.
-        # http://pommedesang.com/efiction/viewstory.php?sid=1234
-        # http://pommedesang.com/sds/viewstory.php?sid=1234
-        self.section=self.parsedUrl.path.split('/',)[1]
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pmds')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        if 'efiction' in self.section:
-            self.dateformat = "%b %d, %Y"
-        else:
-            self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'pommedesang.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/sds/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return r"http://"+self.getSiteDomain()+"/(efiction|sds)?/viewstory.php\?sid=\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/'+self.section+'/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile('viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -1,249 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PonyFictionArchiveNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        # normalized story URL.
-        if "explicit" in self.parsedUrl.netloc:
-            self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-            self.dateformat = "%d/%b/%y"
-        else:
-            self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-            self.dateformat = "%d %b %Y"
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pffa')
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'ponyfictionarchive.net'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=9"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-           
-        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-        for warning in warnings:
-            self.story.addToList('warnings',warning.string)
-        
-        status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-        self.story.setMetadata('status',status.string)
-        
-        section = soup.findAll('span', {'class' : 'General'})[1]            
-        
-        self.story.setMetadata('rating', section.previousSibling.previousSibling.string)
-
-        value = section.nextSibling
-        svalue = ""
-        while not defaultGetattr(value,'class') == 'label':
-            svalue += str(value)
-            value = value.nextSibling
-        self.setDescription(url,svalue)
-        
-        # <span class="label">Rated:</span> NC-17<br /> etc           
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -17,7 +17,6 @@
 
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
@@ -55,7 +54,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
@@ -63,6 +62,11 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','prtky') # XXX
 
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y" # XXX
@@ -72,29 +76,22 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'fanfiction.portkey.org' # XXX
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/story/1234"
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story/1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/story/")+r"\d+(/\d+)?$"
 
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
-    
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         # portkey screws around with using a different URL to set the
         # cookie and it's a pain.  So... cheat!
         if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
             cookie = cl.Cookie(version=0, name='verify17', value='1',
                                port=None, port_specified=False,
                                domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -105,8 +102,9 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
                                comment=None,
                                comment_url=None,
                                rest={'HttpOnly': None},
-                               rfc2109=False) 
-            self.cookiejar.set_cookie(cookie)
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
 
         try:
             data = self._fetchUrl(url)
@@ -254,12 +252,10 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
-
-        data = data.replace("HTML>","div>")
-        
-        soup = bs.BeautifulSoup(data)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 
         #print("soup:%s"%soup)
         tag = soup.find('td', {'class' : 'story'})

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -15,10 +15,8 @@
 # limitations under the License.
 #
 
-# Software: eFiction
 import time
 import logging
-logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -39,10 +37,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
+        self.story.addToList("category","Harry Potter")
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -56,9 +55,8 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.potionsandsnitches.net','potionsandsnitches.net']
 
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234"
+    def getSiteExampleURLs(self):
+        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234 http://potionsandsnitches.net/fanfiction/viewstory.php?sid=5678"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"
@@ -66,7 +64,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logger.debug("URL: "+url)
+        logging.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -84,7 +82,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
+        self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -144,7 +142,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
                 charstext = [char.string for char in chars]
                 for char in charstext:
-                    if "Snape and Harry (required)" in char:
+                    if char == "!Snape and Harry (required)":
                         self.story.addToList('characters',"Snape")
                         self.story.addToList('characters',"Harry")
                     else:
@@ -164,11 +162,12 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                # limit date values, there's some extra chars.
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%b %d %Y"))
             
             if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
 
         try:
             # Find Series name from series URL.
@@ -183,7 +182,6 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -194,7 +192,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         
     def getChapterText(self, url):
 
-        logger.debug('Getting chapter text from: %s' % url)
+        logging.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_potterficscom.py

@@ -1,281 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import datetime
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return PotterFicsComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PotterFicsComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-            
-            # normalized story URL. gets rid of chapter if there, left with chapter index URL
-            nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
-            self._setURL(nurl)
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-        
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','potficscom')
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.potterfics.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://www.potterfics.com/historias/12345 http://www.potterfics.com/historias/12345/capitulo-1 "
-
-    def getSiteURLPattern(self):
-        #http://www.potterfics.com/historias/127583
-        #http://www.potterfics.com/historias/127583/capitulo-1
-        #http://www.potterfics.com/historias/127583/capitulo-4
-        #http://www.potterfics.com/historias/92810              -> Complete story
-        #http://www.potterfics.com/historias/111194             -> Complete, single chap
-        p = re.escape("http://"+self.getSiteDomain()+"/historias/")+\
-            r"(?P<id>\d+)(/capitulo-(?P<ch>\d+))?/?$"
-        return p
-
-    def needToLoginCheck(self, data):
-        # partials used to avoid having to figure out what was wrong
-        # with included utf8 higher chars.
-        if 'Para ver esta historia, por favor inicia tu sesi' in data \
-                or '<script>alert("El nombre de usuario o contrase' in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self,url):
-        params = {}
-
-        if self.password:
-            params['login_usuario'] = self.username
-            params['login_password'] = self.password
-        else:
-            params['login_usuario'] = self.getConfig("username")
-            params['login_password'] = self.getConfig("password")
-        params['login_ck'] = '1'
-
-        loginUrl = 'http://www.potterfics.com/secciones/usuarios/login.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['login_usuario']))
-        d = self._postUrl(loginUrl,params)
-
-        #print("d:%s"%d)
-        if '<script>alert("El nombre de usuario o contrase' in d:
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['login_usuario']))
-            raise exceptions.FailedToLogin(url,params['login_usuario'])
-            return False
-        else:
-            return True
-
-    def extractChapterUrlsAndMetadata(self):
-
-        #this converts '/historias/12345' to 'http://www.potterfics.com/historias/12345'
-        def makeAbsoluteURL(url):
-            if url[0] == '/':
-                url = 'http://'+self.getSiteDomain()+url
-            return url
-
-        #use this to get month numbers from Spanish months
-        SpanishMonths = {
-            'enero'      : '01',
-            'febrero'    : '02',
-            'marzo'      : '03',
-            'abril'      : '04',
-            'mayo'       : '05',
-            'junio'      : '06',
-            'julio'      : '07',
-            'agosto'     : '08',
-            'septiembre' : '09',
-            'octubre'    : '10',
-            'noviembre'  : '11',
-            'diciembre'  : '12'
-            }
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        #print data
-
-        #deal with adult content login
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url,usecache=False)
-            
-        #set constant meta for this site:
-        #Set Language = Spanish
-        self.story.setMetadata('language', 'Spanish')
-        #Set Category = Harry Potter
-        #  This is better done in plugin-defaults.ini and defaults.ini
-        #  by adding a section for this site with the line:
-        #  extracategories:Harry Potter
-        #self.story.addToList('category','Harry Potter')
-        
-        #get the rest of the meta
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        #self closing br and img present!
-        soup = bs.BeautifulSoup(data,selfClosingTags=('br','img'))
-    
-        #we want the second table directly under the body, contains all the metadata
-        table = soup.html.body.findAll('table', recursive=False)[1]
-        #within that, we want the second row, first cell
-        cell = table.tr.findNextSibling('tr').td
-
-        #find first metadata block--isn't first if logged in
-        mb = cell.div.findNextSibling('div',{'align':'left'})
-        #Get meta...
-        self.story.setMetadata('title', stripHTML(mb.b))
-        #strip out brackets on rating
-        self.story.setMetadata('rating', mb.span.string[1:-1])
-        #Completion status is denoted by the presence of this image:
-        if mb.find('img',title="Historia terminada"):
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        #find next metadata block
-        #author details
-        mb = mb.findNextSibling('div')
-        self.story.setMetadata('author', mb.b.a.string.strip())
-        self.story.setMetadata('authorUrl', makeAbsoluteURL(mb.b.a['href']))
-        self.story.setMetadata('authorId', self.story.getMetadata('authorUrl').split('/')[4])
-        #dates and times
-        mb = mb.find('span')
-        #posted/published = Escrita
-        date = mb.find(text=re.compile('Escrita el ')).strip().split()
-        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
-        month = int(SpanishMonths[date[5].lower()])
-        day = int(date[3])
-        time = date[8].split(':')
-        hour = int(time[0])
-        minute = int(time[1])
-        self.story.setMetadata('datePublished', datetime.datetime(year, month, day, hour, minute))
-        #updated = Actualizada
-        date = mb.find(text=re.compile('Actualizada el ')).strip().split()
-        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
-        month = int(SpanishMonths[date[5].lower()])
-        day = int(date[3])
-        time = date[8].split(':')
-        hour = int(time[0])
-        minute = int(time[1])
-        self.story.setMetadata('dateUpdated', datetime.datetime(year, month, day, hour, minute))
-        
-        mb = mb.span.findNextSibling('span').findNextSibling('span')
-        wc = mb.find(text=re.compile(' palabras en total')).strip()
-        self.story.setMetadata('numWords', wc.split()[0])
-        
-        #then we come to categories and genres. Oh dear. On this site, categories hold everything from genre, to ships, to crossovers.
-        #To make things worse, there is also another genre field, which often holds similar/duplicate info. Links to genre pages do not work
-        #though, so perhaps those will be phased out?
-        #for now, put them all into the genre list
-        links = mb.findAll('a',href=re.compile('/(categorias|generos)/\d+'))
-        genlist = [i.string.strip() for i in links]
-        self.story.extendList('genre',genlist)
-        
-        #get the chapter urls
-        #we can go back to the table cell we found before
-        #get its last element and work backwards to find the last ordered list on the page
-        list = cell.contents[len(cell)-1].findPrevious('ol')
-        chapters = []
-        revs = 0
-        chnum = 0
-        for li in list:
-            chnum += 1
-            chTitle = str(chnum) + '. ' + li.a.b.string.strip()
-            chURL = makeAbsoluteURL(li.a['href'])
-            chapters.append((chTitle,chURL))
-            #Get reviews, add to total
-            revs += int(li.div.a.string.split()[0])
-        
-        self.chapterUrls.extend(chapters)
-        self.story.setMetadata('numChapters', len(chapters))
-        self.story.setMetadata('reviews', revs)
-        
-        #Now for the description... this may be tricky...
-        #if it is there (doesn't have to be), it will be before the chapter list,
-        #separated by a horizontal rule, and after the google ad bar
-        
-        #get list's parent div
-        mb = list.parent
-        #get the div before that, will either be the description, or the google ad bar
-        mb = mb.findPreviousSibling('div')
-        if 'google_ad_client' in str(mb):
-            #couldn't find description, leaving it blank
-            pass
-        else:
-            self.setDescription(url,mb)
-        
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr','img'))
-        
-        div = soup.find('div', {'id' : 'cuerpoHistoria'})
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-        
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_potterheadsanonymouscom.py

@@ -1,299 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PotterHeadsAnonymousComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pha')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfic.potterheadsanonymous.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        pagetitle = soup.find('div',{'id':'pagetitle'})
-        
-        ## Title
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -1,254 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PretenderCenterComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PretenderCenterComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ptdc')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'pretendercentre.com'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.pretendercentre.com','pretendercentre.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/missingpieces/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/missingpieces/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/missingpieces/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.findAll('div', {'id' : 'story'})[1]
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_psychficcom.py

@@ -1,249 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PsychFicComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PsychFicComAdapter(BaseSiteAdapter): 
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','psyf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.psychfic.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.text
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_qafficcom.py

@@ -1,265 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return QafFicComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class QafFicComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','atp')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.qaf-fic.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/atp/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=")+r"\d+$"
-       
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=NC-17"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\s+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/atp/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select')
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                # just in case there's tags, like <i> in chapter titles. 
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                break
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
-        for labelspan in labels:
-            label = labelspan.string
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'classification' and value != None:
-                    if "Featured Stories" not in value:
-                        svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
-
-            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            if list.find('a', href=re.compile(r"series.php")) != None:
-                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
-                    # Find Series name from series URL.
-                    series_url = 'http://'+self.host+'/atp/'+series['href']
-                    # use BeautifulSoup HTML parser to make everything easier to find.
-                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-                    i=1
-                    for a in storyas:
-                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                            name=seriessoup.find('div', {'id' : 'pagetitle'})
-                            name.find('a').extract()
-                            self.setSeries(name.text.split(' by[')[0], i)
-                            self.story.setMetadata('seriesUrl',series_url)
-                            i=0
-                            break
-                        i+=1
-                    if i == 0:
-                        break
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-                                     
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)