Update CLI download zip.

Also don't retry 404 errors and bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-23
+version: 4-5-07
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -1,13 +1,28 @@
 #!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 # -*- coding: utf-8 -*-
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import sys
+if sys.version_info >= (2, 7):
+    import logging
+    logger = logging.getLogger(__name__)
+    loghandler=logging.StreamHandler()
+    loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+    logger.addHandler(loghandler)
+    loghandler.setLevel(logging.DEBUG)
+    logger.setLevel(logging.DEBUG)
+
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
@@ -24,11 +39,11 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     calibre utilities to run without needing to load the GUI libraries.
     '''
     name                = 'FanFictionDownLoader'
-    description         = 'UI plugin to download FanFiction stories from various sites.'
+    description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 6, 5)
-    minimum_calibre_version = (0, 8, 57)
+    version             = (1, 8, 26)
+    minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name
@@ -80,11 +95,29 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
         if ac is not None:
             ac.apply_settings()
 
-# For testing, run from command line with this:
-# calibre-debug -e __init__.py
-# 
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    from calibre.gui2.preferences import test_widget
-    app = QApplication([])
-    test_widget('Advanced', 'Plugins')
+    def cli_main(self,argv):
+        # I believe there's no performance hit loading these here when
+        # CLI--it would load everytime anyway.
+        from StringIO import StringIO
+        from calibre.library import db
+        from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
+        from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
+        from calibre.utils.config import prefs as calibre_prefs
+        from optparse import OptionParser      
+
+        parser = OptionParser('%prog --run-plugin '+self.name+' -- [options] <storyurl>')
+        parser.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
+        # parser.add_option('--dont-notify-gui', default=False, action='store_true',
+        #               help=_('Do not notify the running calibre GUI (if any) that the database has'
+        #                      ' changed. Use with care, as it can lead to database corruption!'))
+
+        pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
+                 or not x.startswith('-')]
+        opts, args = parser.parse_args(pargs)
+
+        ffdl_prefs = PrefsFacade(db(path=opts.library_path,
+                                    read_only=True))
+        ffdl_main(argv[1:],
+                  parser=parser,
+                  passed_defaultsini=StringIO(get_resources("defaults.ini")),
+                  passed_personalini=StringIO(ffdl_prefs["personal.ini"]))

calibre-plugin/common_utils.py

@@ -196,7 +196,7 @@ class ImageTitleLayout(QHBoxLayout):
     '''
     A reusable layout widget displaying an image followed by a title
     '''
-    def __init__(self, parent, icon_name, title):
+    def __init__(self, parent, icon_name, title, tooltip=None):
         QHBoxLayout.__init__(self)
         title_image_label = QLabel(parent)
         pixmap = get_pixmap(icon_name)
@@ -217,6 +217,9 @@ class ImageTitleLayout(QHBoxLayout):
         self.addWidget(shelf_label)
         self.insertStretch(-1)
 
+        if tooltip:
+            title_image_label.setToolTip(tooltip)
+            shelf_label.setToolTip(tooltip)
 
 class SizePersistedDialog(QDialog):
     '''
@@ -236,8 +239,8 @@ class SizePersistedDialog(QDialog):
             self.restoreGeometry(self.geom)
 
     def dialog_closing(self, result):
-        geom = bytearray(self.saveGeometry())
-        gprefs[self.unique_pref_name] = geom
+        self.geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = self.geom
 
 
 class ReadOnlyTableWidgetItem(QTableWidgetItem):

calibre-plugin/ffdl_util.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+from StringIO import StringIO
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)
+
+def get_ffdl_personalini():
+    if prefs['includeimages']:
+        # this is a cheat to make it easier for users.
+        return '''[epub]
+include_images:true
+keep_summary_html:true
+make_firstimage_cover:true
+''' + prefs['personal.ini']
+    else:
+        return prefs['personal.ini']
+
+def get_ffdl_config(url,fileform="epub",personalini=None):
+    if not personalini:
+        personalini = get_ffdl_personalini()
+    site='unknown'
+    try:
+        site = adapters.getConfigSectionFor(url)
+    except Exception as e:
+        print("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
+    configuration = Configuration(site,fileform)
+    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+    configuration.readfp(StringIO(personalini))
+
+    return configuration
+
+def get_ffdl_adapter(url,fileform="epub",personalini=None):
+    return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
+    

calibre-plugin/jobs.py

@@ -8,20 +8,23 @@ __copyright__ = '2012, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import time, os, traceback
 
-from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
-from calibre.utils.logging import Log
+from calibre.constants import numeric_version as calibre_version
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
 
+from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config)
 # ------------------------------------------------------------------------------
 #
 #              Functions to perform downloads using worker jobs
@@ -38,18 +41,19 @@ def do_download_worker(book_list, options,
     '''
     server = Server(pool_size=cpus)
 
-    print(options['version'])
+    logger.info(options['version'])
     total = 0
+    alreadybad = []
     # Queue all the jobs
-    print("Adding jobs for URLs:")
+    logger.info("Adding jobs for URLs:")
     for book in book_list:
+        logger.info("%s"%book['url'])
         if book['good']:
-            print("%s"%book['url'])
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',
                     (book,options)]
-            job = ParallelJob('arbitrary',
+            job = ParallelJob('arbitrary_n',
                               "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
                               done=None,
                               args=args)
@@ -59,6 +63,9 @@ def do_download_worker(book_list, options,
             # job._modified_date = modified_date
             # job._existing_isbn = existing_isbn
             server.add_job(job)
+        else:
+            # was already bad before the subprocess ever started.
+            alreadybad.append(book)
     
     # This server is an arbitrary_n job, so there is a notifier available.
     # Set the % complete to a small number to avoid the 'unavailable' indicator
@@ -81,21 +88,18 @@ def do_download_worker(book_list, options,
         book_id = job._book['calibre_id']
         #title = job._title
         count = count + 1
-        notification(float(count)/total, 'Downloaded Story')
+        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
         # Add this job's output to the current log
-        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        print(job.details)
+        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        logger.info(job.details)
+
+
 
         if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            print("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    print(book['title'])
-            print("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    print(book['title'])
+            logger.info("\nSuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\nUnsuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
             break
 
     server.close()
@@ -103,7 +107,7 @@ def do_download_worker(book_list, options,
     # return the book list as the job result
     return book_list
 
-def do_download_for_worker(book,options):
+def do_download_for_worker(book,options,notification=lambda x,y:x):
     '''
     Child job, to extract isbn from formats for this specific book,
     when run as a worker job
@@ -111,40 +115,62 @@ def do_download_for_worker(book,options):
     try:
         book['comment'] = 'Download started...'
 
-        ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        configuration = get_ffdl_config(book['url'],
+                                        options['fileform'],
+                                        options['personal.ini'])
         
         if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-            ffdlconfig.set("overrides","never_make_cover","true")
+            configuration.set("overrides","never_make_cover","true")
+
+        # images only for epub, even if the user mistakenly turned it
+        # on else where.
+        if options['fileform'] not in ("epub","html"):
+            configuration.set("overrides","include_images","false")
         
-        adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
+        adapter = adapters.getAdapter(configuration,book['url'])
         adapter.is_adult = book['is_adult'] 
         adapter.username = book['username'] 
         adapter.password = book['password']
+        adapter.setChaptersRange(book['begin'],book['end'])
         
         story = adapter.getStoryMetadataOnly()
         if 'calibre_series' in book:
-            # print("calibre_series:%s [%d]"%book['calibre_series'])
             adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
-        # else:
-        #     print("no calibre_series")
-        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+            
+        # set PI version instead of default.
+        if 'version' in options:
+            story.setMetadata('version',options['version'])
+            
+        writer = writers.getWriter(options['fileform'],configuration,adapter)
 
         outfile = book['outfile']
 
         ## No need to download at all.  Shouldn't ever get down here.
         if options['collision'] in (CALIBREONLY):
-            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
             book['comment'] = 'Metadata collected.'
             
         ## checks were done earlier, it's new or not dup or newer--just write it.
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
 
-            adapter.setChaptersRange(book['begin'],book['end'])
+            # preserve logfile even on overwrite.
+            if 'epub_for_update' in book:
+                (urlignore,
+                 chaptercountignore,
+                 oldchaptersignore,
+                 oldimgsignore,
+                 oldcoverignore,
+                 calibrebookmarkignore,
+                 # only logfile set in adapter, so others aren't used.
+                 adapter.logfile) = get_update_data(book['epub_for_update'])
+
+                # change the existing entries id to notid so
+                # write_epub writes a whole new set to indicate overwrite.
+                if adapter.logfile:
+                    adapter.logfile = adapter.logfile.replace("span id","span notid")
             
-            print("write to %s"%outfile)
+            logger.info("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
             
@@ -153,7 +179,7 @@ def do_download_for_worker(book,options):
 
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
-            urlchaptercount = int(story.getMetadata('numChapters'))
+            urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
             (url,
              chaptercount,
              adapter.oldchapters,
@@ -162,14 +188,46 @@ def do_download_for_worker(book,options):
              adapter.calibrebookmark,
              adapter.logfile) = get_update_data(book['epub_for_update'])
 
-            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            print("write to %s"%outfile)
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if options['collision'] == UPDATE:
+                if chaptercount == urlchaptercount:
+                    book['comment']="Already contains %d chapters.  Reuse as is."%chaptercount
+                    book['outfile'] = book['epub_for_update'] # for anthology merge ops.
+                    return book
+
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if chaptercount > urlchaptercount:
+                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+
+            if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
+                    and adapter.getConfig("do_update_hook"):
+                chaptercount = adapter.hookForUpdates(chaptercount)
+
+            logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            logger.info("write to %s"%outfile)
 
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             
             book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
                 (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
         
+        if options['smarten_punctuation'] and options['fileform'] == "epub" \
+                and calibre_version >= (0, 9, 39):
+            # do smarten_punctuation from calibre's polish feature
+            from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+            from calibre.utils.logging import Log
+            from collections import namedtuple
+        
+            data = {'smarten_punctuation':True}
+            opts = ALL_OPTS.copy()
+            opts.update(data)
+            O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+            opts = O(**opts)
+            
+            log = Log(level=Log.DEBUG)
+            # report = []
+            polish({outfile:outfile}, opts, log, logger.info) # report.append
+        
     except NotGoingToDownload as d:
         book['good']=False
         book['comment']=unicode(d)
@@ -180,8 +238,8 @@ def do_download_for_worker(book,options):
         book['comment']=unicode(e)
         book['icon']='dialog_error.png'
         book['status'] = 'Error'
-        print("Exception: %s:%s"%(book,unicode(e)))
+        logger.info("Exception: %s:%s"%(book,unicode(e)))
         traceback.print_exc()
-
+        
     #time.sleep(10)
     return book

calibre-plugin/prefs.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import copy
+
+from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import OVERWRITE
+from calibre_plugins.fanfictiondownloader_plugin.common_utils import get_library_uuid
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['suppressauthorsort'] = False
+default_prefs['suppresstitlesort'] = False
+default_prefs['mark'] = False
+default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['checkforseriesurlid'] = True
+default_prefs['checkforurlchange'] = True
+default_prefs['injectseries'] = False
+default_prefs['smarten_punctuation'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+default_prefs['allow_custcol_from_ini'] = True
+
+default_prefs['std_cols_newonly'] = {}
+
+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
+def set_library_config(library_config,db):
+    db.prefs.set_namespaced(PREFS_NAMESPACE,
+                            PREFS_KEY_SETTINGS,
+                            library_config)
+
+def get_library_config(db):
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        #print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config,db)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        #print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def _get_db(self):
+        if self.passed_db:
+            return self.passed_db
+        else:
+            # In the GUI plugin we want current db so we detect when
+            # it's changed.  CLI plugin calls need to pass db in.
+            return get_gui().current_db
+    
+    def __init__(self,passed_db=None):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
+        self.passed_db=passed_db
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(self._get_db())
+        if self.current_prefs == None or self.libraryid != libraryid:
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config(self._get_db())
+        return self.current_prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        # self._save_prefs(prefs)
+
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        if k in prefs:
+            del prefs[k]
+
+    def save_to_db(self):
+        set_library_config(self._get_prefs(),self._get_db())
+        
+prefs = PrefsFacade()
+

downloader.py

@@ -15,8 +15,6 @@
 # limitations under the License.
 #
 
-import logging
-
 import sys, os
 from os.path import normpath, expanduser, isfile, join
 from StringIO import StringIO
@@ -25,10 +23,35 @@ import getpass
 import string
 import ConfigParser
 from subprocess import call
+import pprint
 
-from fanficdownloader import adapters,writers,exceptions
-from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-from fanficdownloader.geturls import get_urls_from_page
+import logging
+if sys.version_info >= (2, 7):
+    # suppresses default logger.  Logging is setup in fanficdownload/__init__.py so it works in calibre, too.
+    rootlogger = logging.getLogger()
+    loghandler=logging.NullHandler()
+    loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
+    rootlogger.addHandler(loghandler)
+
+try:
+    from calibre.constants import numeric_version as calibre_version
+    is_calibre = True
+except:
+    is_calibre = False
+
+# using try/except directly was masking errors during development.
+if is_calibre:
+    # running under calibre
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
+else:
+    from fanficdownloader import adapters,writers,exceptions
+    from fanficdownloader.configurable import Configuration
+    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficdownloader.geturls import get_urls_from_page
+    
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
@@ -41,15 +64,23 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     del writer
     return output_filename
 
-def main():
+def main(argv,
+         parser=None,
+         passed_defaultsini=None,
+         passed_personalini=None):
    # read in args, anything starting with -- will be treated as --<varible>=<value>
-   usage = "usage: %prog [options] storyurl"
-   parser = OptionParser(usage)
+   if not parser:
+       parser = OptionParser("usage: %prog [options] storyurl")
    parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), text or html", metavar="FORMAT")
+                     help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT")
+
+   if passed_defaultsini:
+       config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini"
+   else:
+       config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini"
    parser.add_option("-c", "--config",
                      action="append", dest="configfile", default=None,
-                     help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG")
+                     help=config_help, metavar="CONFIG")
    parser.add_option("-b", "--begin", dest="begin", default=None,
                      help="Begin with Chapter START", metavar="START")
    parser.add_option("-e", "--end", dest="end", default=None,
@@ -62,44 +93,86 @@ def main():
                      help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
+                     help="Update an existing epub with new chapters, give epub filename instead of storyurl.",)
    parser.add_option("--update-cover",
                      action="store_true", dest="updatecover",
                      help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
-                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
+                     help="Force overwrite of an existing epub, download and overwrite all chapters.",)
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
+   parser.add_option("-n", "--normalize-list",
+                     action="store_true", dest="normalize",default=False,
+                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
+   parser.add_option("-s", "--sites-list",
+                     action="store_true", dest="siteslist",default=False,
+                     help="Get list of valid story URLs examples.",)
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug",
                      help="Show debug output while downloading.",)
    
-   (options, args) = parser.parse_args()
-
-   if options.debug:
-       logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
-   else:
-       logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+   (options, args) = parser.parse_args(argv)
 
+   if not options.debug:
+       logger = logging.getLogger("fanficdownloader")
+       logger.setLevel(logging.INFO)
    
-   if len(args) != 1:
+   if not options.siteslist and len(args) != 1:
        parser.error("incorrect number of arguments")
 
+   if options.siteslist:
+       for (site,examples) in adapters.getSiteExamples():
+           print("\n====%s====\n\nExample URLs:"%site)
+           for u in examples:
+               print("  * %s"%u)
+       return
+
    if options.update and options.format != 'epub':
        parser.error("-u/--update-epub only works with epub")
 
-   config = ConfigParser.SafeConfigParser()
+   ## Attempt to update an existing epub.
+   chaptercount = None
+   output_filename = None
+   if options.update:
+       try:
+           (url,chaptercount) = get_dcsource_chaptercount(args[0])
+           if not url:
+               print "No story URL found in epub to update."
+               return
+           print "Updating %s, URL: %s" % (args[0],url)
+           output_filename = args[0]
+       except:
+           # if there's an error reading the update file, maybe it's a URL?
+           # we'll look for an existing outputfile down below.
+           url = args[0]
+   else:
+       url = args[0]
+
+   try:
+       configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+   except exceptions.UnknownSite, e:
+       if options.list or options.normalize:
+           # list for page doesn't have to be a supported site.
+           configuration = Configuration("test1.com",options.format)
+       else:
+           raise e
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
+
+   if passed_defaultsini:
+       configuration.readfp(passed_defaultsini)
    
    if isfile(join(homepath,"defaults.ini")):
        conflist.append(join(homepath,"defaults.ini"))
    if isfile("defaults.ini"):
        conflist.append("defaults.ini")
        
+   if passed_personalini:
+       configuration.readfp(passed_personalini)
+   
    if isfile(join(homepath,"personal.ini")):
        conflist.append(join(homepath,"personal.ini"))
    if isfile("personal.ini"):
@@ -109,42 +182,52 @@ def main():
        conflist.extend(options.configfile)
       
    logging.debug('reading %s config file(s), if present'%conflist)
-   config.read(conflist)
+   configuration.read(conflist)
 
    try:
-       config.add_section("overrides")
+       configuration.add_section("overrides")
    except ConfigParser.DuplicateSectionError:
        pass
 
    if options.force:
-       config.set("overrides","always_overwrite","true")
+       configuration.set("overrides","always_overwrite","true")
 
+   if options.update and chaptercount:
+       configuration.set("overrides","output_filename",output_filename)
+       
    if options.update and not options.updatecover:
-       config.set("overrides","never_make_cover","true")
+       configuration.set("overrides","never_make_cover","true")
+
+   # images only for epub, even if the user mistakenly turned it
+   # on else where.
+   if options.format not in ("epub","html"):
+       configuration.set("overrides","include_images","false")
        
    if options.options:
        for opt in options.options:
            (var,val) = opt.split('=')
-           config.set("overrides",var,val)
+           configuration.set("overrides",var,val)
 
-   if options.list:
-       retlist = get_urls_from_page(args[0], config)
-       print "\n".join(retlist)
-               
+   if options.list or options.normalize:
+       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
+       print "\n".join(retlist)               
        return
 
    try:
-       ## Attempt to update an existing epub.
-       if options.update:
-           (url,chaptercount) = get_dcsource_chaptercount(args[0])
-           print "Updating %s, URL: %s" % (args[0],url)
-           output_filename = args[0]
-           config.set("overrides","output_filename",args[0])
-       else:
-           url = args[0]
-
-       adapter = adapters.getAdapter(config,url,options.format)
+       adapter = adapters.getAdapter(configuration,url)
+       adapter.setChaptersRange(options.begin,options.end)
 
+       # check for updating from URL (vs from file)
+       if options.update and not chaptercount:
+           try:
+               writer = writers.getWriter("epub",configuration,adapter)
+               output_filename=writer.getOutputFileName()
+               (noturl,chaptercount) = get_dcsource_chaptercount(output_filename)
+               print "Updating %s, URL: %s" % (output_filename,url)
+           except:
+               options.update = False
+               pass
+       
        ## Check for include_images and absence of PIL, give warning.
        if adapter.getConfig('include_images'):
            try:
@@ -158,7 +241,6 @@ def main():
                    print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
                    if not sys.stdin.readline().strip().lower().startswith('y'):
                        return
-               
 
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
@@ -181,36 +263,39 @@ def main():
 
        if options.update and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
-           
+
            if chaptercount == urlchaptercount and not options.metaonly:
-               print "%s already contains %d chapters." % (args[0],chaptercount)
+               print "%s already contains %d chapters." % (output_filename,chaptercount)
            elif chaptercount > urlchaptercount:
-               print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
+               print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
+           elif chaptercount == 0:
+               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
+               # update now handled by pre-populating the old
+               # images and chapters in the adapter rather than
+               # merging epubs.
+               (url,
+                chaptercount,
+                adapter.oldchapters,
+                adapter.oldimgs,
+                adapter.oldcover,
+                adapter.calibrebookmark,
+                adapter.logfile) = get_update_data(output_filename)
+
                print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
-               if not options.metaonly:
 
-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(args[0])
+               if not (options.update and chaptercount == urlchaptercount) \
+                       and adapter.getConfig("do_update_hook"):
+                   chaptercount = adapter.hookForUpdates(chaptercount)
 
-                   writeStory(config,adapter,"epub")
+               writeStory(configuration,adapter,"epub")
                    
        else:
            # regular download
            if options.metaonly:
-               print adapter.getStoryMetadataOnly()
+               pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
            
-           adapter.setChaptersRange(options.begin,options.end)
-           
-           output_filename=writeStory(config,adapter,options.format,options.metaonly)
+           output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        
        if not options.metaonly and adapter.getConfig("post_process_cmd"):
            metadata = adapter.story.metadata
@@ -230,5 +315,5 @@ def main():
 if __name__ == "__main__":
     #import time
     #start = time.time()
-    main()
+    main(sys.argv[1:])
     #print("Total time seconds:%f"%(time.time()-start))

example.ini

@@ -1,40 +1,103 @@
 ## This is an example of what your personal configuration might look
-## like.
+## like.  Uncomment options by removing the '#' in front of them.
 
 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
 #is_adult:true
 
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+#strip_chapter_numbers:true
+#add_chapter_numbers:true
+
+[epub]
+## include images from img tags in the body and summary of stories.
+## Images will be converted to jpg for size if possible.  Images work
+## in epub format only.  To get mobi or other format with images,
+## download as epub and use Calibre to convert.
+#include_images:true
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:true
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover:true
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+#image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+
 ## Most common, I expect will be using this to save username/passwords
-## for different sites.
+## for different sites.  Here are a few examples.  See defaults.ini
+## for the full list.
+
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
+## default is false
+#collect_series: true
 
 [www.ficwad.com]
 #username:YourUsername
 #password:YourPassword
 
+[www.twiwrite.net]
+#username:YourName
+#password:yourpassword
+## default is false
+#collect_series: true
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## The [defaults] section here will override the system [defaults],
-## but not format, site for site:format sections.
-[defaults]
-## Directories only useful in commandline or zip files.
-#output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
-#output_filename: books/${site}/${authorId}/${title}-${storyId}${formatext}
+[www.thewriterscoffeeshop.com]
+#username:YourName
+#password:yourpassword
+#is_adult:true
+## default is false
+#collect_series: true
+
+[www.fictionalley.org]
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+#is_adult:true
+
+[www.fimfiction.net]
+#is_adult:true
+#fail_on_password: false
+
+[www.tthfanfic.org]
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
 
-## For example, zip_output here will turn on zip for html and txt, but
-## not epub because the system [epub] section explicitly says
-## zip_output: false (epubs *are* specially formated zip files.)
-#zip_output: true
-#zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 
 ## This section will override anything in the system defaults or other
 ## sections here.
 [overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true

fanficdownloader/HtmlTagStack.py

@@ -0,0 +1,57 @@
+# coding: utf-8
+
+import re
+import codecs
+
+stack = []
+
+def get_end_tag(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'.*<([^\ >]+).*', r'</\1>', tag)
+    return u''
+
+def get_tag_name(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'</*([^\ >]+).*', r'\1', tag)
+    return u''
+
+def push(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        stack.append(tag)
+
+def pop():
+    if len(stack) > 0:
+        return stack.pop()
+    return u''
+
+def pop_end_tag():
+    return unicode(get_end_tag(pop()))
+
+def spool_end():
+    html = u''
+    for tag in reversed(stack):
+        html += get_end_tag(tag)
+    return html
+
+def spool_start():
+    html = u''
+    for item in stack:
+        html += item
+    return html
+
+def has_elements():
+    return len(stack) > 0
+
+def get_last():
+    # t = pop()
+    # push(t)
+    # return t
+    if len(stack) > 0:
+        return stack[len(stack)-1]
+    return u''
+
+def flush():
+    del stack[:]
+
+def get_stack():
+    return stack

fanficdownloader/__init__.py

@@ -1 +1,19 @@
 # -*- coding: utf-8 -*-
+
+try:
+    # just a way to switch between web service and CLI/PI
+    import google.appengine.api 
+except:
+    try: # just a way to switch between CLI and PI
+        import calibre.constants
+    except:
+        import sys
+        if sys.version_info >= (2, 7):
+            import logging
+            logger = logging.getLogger(__name__)
+            loghandler=logging.StreamHandler()
+            loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+            logger.addHandler(loghandler)
+            loghandler.setLevel(logging.DEBUG)
+            logger.setLevel(logging.DEBUG)
+

fanficdownloader/adapters/__init__.py

@@ -1,158 +1,244 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob, types
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-from .. import exceptions as exceptions
-
-## must import each adapter here.
-
-import adapter_test1
-import adapter_fanfictionnet
-import adapter_castlefansorg
-import adapter_fictionalleyorg
-import adapter_fictionpresscom
-import adapter_ficwadcom
-import adapter_fimfictionnet
-import adapter_harrypotterfanfictioncom
-import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
-import adapter_tenhawkpresentscom
-import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
-import adapter_tthfanficorg
-import adapter_twilightednet
-import adapter_twiwritenet
-import adapter_whoficcom
-import adapter_siyecouk
-import adapter_archiveofourownorg
-import adapter_ficbooknet
-import adapter_portkeyorg
-import adapter_mugglenetcom
-import adapter_hpfandomnet
-import adapter_thequidditchpitchorg
-import adapter_nfacommunitycom
-import adapter_midnightwhispersca
-import adapter_ksarchivecom
-import adapter_archiveskyehawkecom
-import adapter_squidgeorgpeja
-import adapter_libraryofmoriacom
-import adapter_wraithbaitcom
-import adapter_checkmatedcom
-import adapter_chaossycophanthexcom
-import adapter_dramioneorg
-import adapter_erosnsapphosycophanthexcom
-import adapter_lumossycophanthexcom
-import adapter_occlumencysycophanthexcom
-import adapter_phoenixsongnet
-import adapter_walkingtheplankorg
-import adapter_ashwindersycophanthexcom
-import adapter_thehexfilesnet
-import adapter_dokugacom
-import adapter_iketernalnet
-import adapter_onedirectionfanfictioncom
-import adapter_prisonbreakficnet
-import adapter_storiesofardacom
-import adapter_yourfanfictioncom
-import adapter_samdeanarchivenu
-import adapter_destinysgatewaycom
-import adapter_ncisfictioncom
-import adapter_stargateatlantisorg
-import adapter_thealphagatecom
-import adapter_fanfiktionde
-import adapter_ponyfictionarchivenet
-import adapter_sg1heliopoliscom
-import adapter_ncisficcom
-import adapter_nationallibrarynet
-import adapter_themasquenet
-import adapter_pretendercentrecom
-import adapter_darksolaceorg
-import adapter_finestoriescom
-import adapter_hpfanficarchivecom
-import adapter_svufictioncom
-import adapter_twilightarchivescom
-import adapter_wizardtalesnet
-import adapter_nhamagicalworldsus
-import adapter_hlfictionnet
-import adapter_grangerenchantedcom
-import adapter_dracoandginnycom
-import adapter_scarvesandcoffeenet
-import adapter_thepetulantpoetesscom
-import adapter_wolverineandroguecom
-
-## This bit of complexity allows adapters to be added by just adding
-## importing.  It eliminates the long if/else clauses we used to need
-## to pick out the adapter.
-    
-## List of registered site adapters.
-__class_list = []
-
-def imports():
-    for name, val in globals().items():
-        if isinstance(val, types.ModuleType):
-            yield val.__name__
-
-for x in imports():
-    if "fanficdownloader.adapters.adapter_" in x:
-        #print x
-        __class_list.append(sys.modules[x].getClass())
-            
-def getAdapter(config,url,fileform=None):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    logging.debug("trying url:"+url)
-    cls = getClassFor(domain)
-    if not cls and domain.startswith("www."):
-        domain = domain.replace("www.","")
-        logging.debug("trying site:without www: "+domain)
-        cls = getClassFor(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
-    if not cls:
-        logging.debug("trying site:www."+domain)
-        cls = getClassFor("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        adapter.setSectionOrder(adapter.getConfigSection(),fileform)
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getConfigSections():
-    return [cls.getConfigSection() for cls in __class_list]
-
-def getClassFor(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, re, sys, glob, types
+from os.path import dirname, basename, normpath
+import logging
+import urlparse as up
+
+logger = logging.getLogger(__name__)
+
+from .. import exceptions as exceptions
+from ..configurable import Configuration
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
+import adapter_archiveofourownorg
+import adapter_ficbooknet
+import adapter_portkeyorg
+import adapter_mugglenetcom
+import adapter_hpfandomnet
+import adapter_thequidditchpitchorg
+import adapter_nfacommunitycom
+import adapter_midnightwhispersca
+import adapter_ksarchivecom
+import adapter_archiveskyehawkecom
+import adapter_squidgeorgpeja
+import adapter_libraryofmoriacom
+import adapter_wraithbaitcom
+import adapter_checkmatedcom
+import adapter_chaossycophanthexcom
+import adapter_dramioneorg
+import adapter_erosnsapphosycophanthexcom
+import adapter_lumossycophanthexcom
+import adapter_occlumencysycophanthexcom
+import adapter_phoenixsongnet
+import adapter_walkingtheplankorg
+import adapter_ashwindersycophanthexcom
+import adapter_thehexfilesnet
+import adapter_dokugacom
+import adapter_iketernalnet
+import adapter_onedirectionfanfictioncom
+import adapter_storiesofardacom
+import adapter_samdeanarchivenu
+import adapter_destinysgatewaycom
+import adapter_ncisfictionnet
+import adapter_stargateatlantisorg
+import adapter_thealphagatecom
+import adapter_fanfiktionde
+import adapter_ponyfictionarchivenet
+import adapter_sg1heliopoliscom
+import adapter_ncisficcom
+import adapter_nationallibrarynet
+import adapter_themasquenet
+import adapter_pretendercentrecom
+import adapter_darksolaceorg
+import adapter_finestoriescom
+import adapter_hpfanficarchivecom
+import adapter_twilightarchivescom
+import adapter_wizardtalesnet
+import adapter_nhamagicalworldsus
+import adapter_hlfictionnet
+import adapter_grangerenchantedcom
+import adapter_dracoandginnycom
+import adapter_scarvesandcoffeenet
+import adapter_thepetulantpoetesscom
+import adapter_wolverineandroguecom
+import adapter_sinfuldesireorg
+import adapter_merlinficdtwinscouk
+import adapter_thehookupzonenet
+import adapter_bloodtiesfancom
+import adapter_indeathnet
+import adapter_qafficcom
+import adapter_efpfanficnet
+import adapter_potterficscom
+import adapter_efictionestelielde
+import adapter_dotmoonnet
+import adapter_pommedesangcom
+import adapter_restrictedsectionorg
+import adapter_imagineeficcom
+import adapter_buffynfaithnet
+import adapter_psychficcom
+import adapter_hennethannunnet
+import adapter_tokrafandomnetcom
+import adapter_netraptororg
+import adapter_asr3slashzoneorg
+import adapter_nickandgregnet
+import adapter_potterheadsanonymouscom
+import adapter_simplyundeniablecom
+import adapter_scarheadnet
+import adapter_fictionpadcom
+import adapter_storiesonlinenet
+import adapter_trekiverseorg
+import adapter_literotica
+import adapter_voracity2eficcom
+import adapter_spikeluvercom
+import adapter_bloodshedversecom
+import adapter_nocturnallightnet
+import adapter_fanfichu
+import adapter_fanfictioncsodaidokhu
+import adapter_fictionmaniatv
+
+## This bit of complexity allows adapters to be added by just adding
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
+    
+## List of registered site adapters.
+__class_list = []
+__domain_map = {}
+
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        cls = sys.modules[x].getClass()
+        __class_list.append(cls)
+        for site in cls.getAcceptDomains():
+            __domain_map[site]=cls
+
+def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
+    if not getNormalStoryURL.__dummyconfig:
+        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
+    # pulling up an adapter is pretty low over-head.  If
+    # it fails, it's a bad url.
+    try:
+        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
+        url = adapter.url
+        site = adapter.getSiteDomain()
+        del adapter
+        return (url,site)
+    except:
+        return None
+
+# kludgey function static/singleton
+getNormalStoryURL.__dummyconfig = None
+
+def getAdapter(config,url,anyurl=False):
+
+    #logger.debug("trying url:"+url)
+    (cls,fixedurl) = getClassFor(url)
+    #logger.debug("fixedurl:"+fixedurl)
+    if cls:
+        if anyurl:
+            fixedurl = cls.getSiteExampleURLs().split()[0]
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getConfigSections():
+    return [cls.getConfigSection() for cls in __class_list]
+
+def getSiteExamples():
+    l=[]
+    for cls in sorted(__class_list, key=lambda x : x.getConfigSection()):
+        l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
+    return l
+
+def getConfigSectionFor(url):
+    (cls,fixedurl) = getClassFor(url)
+    if cls:
+        return cls.getConfigSection()
+    
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getClassFor(url):
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
+    if fixedurl.startswith("//"):
+        fixedurl = "http:%s"%url
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    cls = getClassFromList(domain)
+    if not cls and domain.startswith("www."):
+        domain = domain.replace("www.","")
+        #logger.debug("trying site:without www: "+domain)
+        cls = getClassFromList(domain)
+        fixedurl = re.sub(r"^http(s?)://www\.",r"http\1://",fixedurl)
+    if not cls:
+        #logger.debug("trying site:www."+domain)
+        cls = getClassFromList("www."+domain)
+        fixedurl = re.sub(r"^http(s?)://",r"http\1://www.",fixedurl)
+
+    if cls:
+        fixedurl = cls.stripURLParameters(fixedurl)
+    
+    return (cls,fixedurl)
+    
+def getClassFromList(domain):
+    try:
+        return __domain_map[domain]
+    except KeyError:
+        pass # return none.

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,12 +38,11 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Star Trek")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -52,6 +52,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.adastrafanfic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -66,7 +67,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             addurl=""
             
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -89,7 +90,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php"))
@@ -195,6 +196,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -205,7 +207,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # problems with some stories, but only in calibre.  I suspect

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#  -*- coding: utf-8 -*-
 
 # Copyright 2011 Fanficdownloader team
 #
@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -30,6 +31,8 @@ def getClass():
     return ArchiveOfOurOwnOrgAdapter
 
 
+logger = logging.getLogger(__name__)
+
 class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -48,13 +51,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
         else:
@@ -75,15 +78,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         return 'archiveofourown.org'
 
     @classmethod
-    def getAcceptDomains(cls):
-        return ['www.archiveofourown.org','archiveofourown.org']
-
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
+        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)(/chapters/\d+)?/?$"
+        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -108,14 +109,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         params['authenticity_token'] = data.split('input name="authenticity_token" type="hidden" value="')[1].split('" /></div>')[0]
 
         loginUrl = 'http://' + self.getSiteDomain() + '/user_sessions'
-        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['user_session[login]']))
     
         d = self._postUrl(loginUrl, params)
-        #logging.info(d)
+        #logger.info(d)
     
         if "Successfully logged in" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['user_session[login]']))
             raise exceptions.FailedToLogin(url,params['user_session[login]'])
             return False
@@ -132,8 +133,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
         metaurl = self.url+addurl
         url = self.url+'/navigate'+addurl
-        logging.info("url: "+url)
-        logging.info("metaurl: "+metaurl)
+        logger.info("url: "+url)
+        logger.info("metaurl: "+metaurl)
 
         try:
             data = self._fetchUrl(url)
@@ -144,10 +145,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
+                raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
+        if "Sorry, we couldn&#x27;t find the work you were looking for." in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+            
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url,data)
@@ -162,36 +166,46 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r"^/works/\d+$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
 		
         # Find authorid and URL from... author url.
         alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
         if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
             self.story.setMetadata('author','Anonymous')
-            self.story.setMetadata('authorUrl',self.url)
+            self.story.setMetadata('authorUrl','http://archiveofourown.org/')
+            self.story.setMetadata('authorId','0')
         else:
             for a in alist:
                 self.story.addToList('authorId',a['href'].split('/')[2])
                 self.story.addToList('authorUrl','http://'+self.host+a['href'])
                 self.story.addToList('author',a.text)
 
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on AO3 it's
+        # possible for authors to insert new chapters out-of-order or
+        # change the dates of earlier ones by editing them--That WILL
+        # break epub update.
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
         self.story.setMetadata('numChapters',len(chapters))
-        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
-            else:
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
+        else:
+            for index, chapter in enumerate(chapters):
+                # strip just in case there's tags, like <i> in chapter titles.
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-
-
+                # (2013-09-21)
+                date = stripHTML(chapter.findNext('span'))[1:-1]
+                chapterDate = makeDate(date,self.dateformat)
+                if newestChapter == None or chapterDate > newestChapter:
+                    newestChapter = chapterDate
+                    self.newestChapterNum = index
 
         a = metasoup.find('blockquote',{'class':'userstuff'})
         if a != None:
-            self.setDescription(url,a.text)
+            self.setDescription(url,a)
             #self.story.setMetadata('description',a.text)
 		
         a = metasoup.find('dd',{'class':"rating tags"})
@@ -201,27 +215,28 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         a = metasoup.find('dd',{'class':"fandom tags"})
         fandoms = a.findAll('a',{'class':"tag"})
         for fandom in fandoms:
+            self.story.addToList('fandoms',fandom.string)
             self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
         if a != None:
             warnings = a.findAll('a',{'class':"tag"})
             for warning in warnings:
-                if warning.string == "Author Chose Not To Use Archive Warnings":
-                    warning.string = "No Archive Warnings Apply"
-                if warning.string != "No Archive Warnings Apply":
-                    self.story.addToList('warnings',warning.string)
+                self.story.addToList('warnings',warning.string)
 		
         a = metasoup.find('dd',{'class':"freeform tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
+                self.story.addToList('freeformtags',genre.string)
                 self.story.addToList('genre',genre.string)
+                
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
                 if genre != "Gen":
+                    self.story.addToList('ao3categories',genre.string)
                     self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
@@ -229,13 +244,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chars = a.findAll('a',{'class':"tag"})
             for char in chars:
                 self.story.addToList('characters',char.string)
+                
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
-            chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
-                self.story.addToList('characters',char.string)
+            ships = a.findAll('a',{'class':"tag"})
+            for ship in ships:
+                self.story.addToList('ships',ship.string)
+		
+        a = metasoup.find('dd',{'class':"collections"})
+        if a != None:
+            collections = a.findAll('a')
+            for collection in collections:
+                self.story.addToList('collections',collection.string)
 		
-
         stats = metasoup.find('dl',{'class':'stats'})
         dt = stats.findAll('dt')
         dd = stats.findAll('dd')
@@ -246,6 +267,18 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             if 'Words:' in label:
                 self.story.setMetadata('numWords', value)
 				
+            if 'Comments:' in label:
+                self.story.setMetadata('comments', value)
+				
+            if 'Kudos:' in label:
+                self.story.setMetadata('kudos', value)
+				
+            if 'Hits:' in label:
+                self.story.setMetadata('hits', value)
+				
+            if 'Bookmarks:' in label:
+                self.story.setMetadata('bookmarks', value)
+				
             if 'Chapters:' in label:
                 if value.split('/')[0] == value.split('/')[1]:
                     self.story.setMetadata('status', 'Completed')
@@ -264,45 +297,58 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
 		
-        try:
-            # Find Series name from series URL.
-            a = metasoup.find('dd',{'class':"series"})
-            b = a.find('a', href=re.compile(r"/series/\d+"))
-            series_name = b.string
-            series_url = 'http://'+self.host+'/fanfic/'+b['href']
-            series_index = int(a.text.split(' ')[1])
-            self.setSeries(series_name, series_index)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        # Find Series name from series URL.
+        ddseries = metasoup.find('dd',{'class':"series"})
 
+        if ddseries:
+            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
+                series_name = stripHTML(a)
+                series_url = 'http://'+self.host+a['href']
+                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
+                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
+                self.story.setMetadata('series%02dUrl'%i,series_url)
+                if i == 0:
+                    self.setSeries(series_name, series_index)
+                    self.story.setMetadata('seriesUrl',series_url)
+
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
             if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    chapter.append("<b>Author's Note:</b>")
+                    chapter.append(headnotes)
         
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                chapter.append("<b>Summary for the Chapter:</b>")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
             if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    chapter.append("<b>Notes for the Chapter:</b>")
+                    chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
         chtext = text.find('h3', {'class' : "landmark heading"})
@@ -310,17 +356,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chtext.extract()
         chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(footnotes)
 			
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
@@ -65,20 +66,22 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archive.skyehawke.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['archive.skyehawke.com','www.skyehawke.com']
+
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
+        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
-
-
-        
+        return re.escape("http://")+r"(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -98,7 +101,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=a.text.split('"')[1]
+        title=stripHTML(a).split('"')[1]
         self.story.setMetadata('title',title)
         
         # Find authorid and URL from... author url.
@@ -151,7 +154,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
                 rating.find('br').replaceWith('split')
                 rating=rating.text.replace("This story is rated",'').split('split')[0]
                 self.story.setMetadata('rating',rating)
-                logging.debug(self.story.getMetadata('rating'))
+                logger.debug(self.story.getMetadata('rating'))
 			
                 warnings=box.find('ol')
                 if warnings != None:
@@ -177,7 +180,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','asph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -70,6 +66,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'ashwinder.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -99,13 +96,13 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Logout" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -117,7 +114,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -160,7 +157,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 
@@ -242,7 +239,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
 

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return Asr3SlashzoneOrgAdapter
+
+class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','asr3')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'asr3.slashzone.org'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
+
+        categorys = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for category in categorys:
+            self.story.addToList('category',category.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        ships = smalldiv.parent.findAll('a',href=re.compile(r'browse\.php\?type=class&type_id=2&classid=1'))
+        for ship in ships:
+            self.story.addToList('ships',ship.string)
+
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
+
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -0,0 +1,192 @@
+from datetime import timedelta
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return BloodshedverseComAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class BloodshedverseComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'bvc'
+    SITE_DOMAIN = 'bloodshedverse.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
+
+    STARTED_DATETIME_FORMAT = '%m/%d/%Y'
+    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_no = query_data['no'][0]
+
+        self.story.setMetadata('storyId', story_no)
+        self._setURL(self.READ_URL_TEMPLATE % story_no)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return BloodshedverseComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL + 'stories.php?go=') + r'(read|chapters)\&no=\d+$'
+
+    # Override stripURLParameters so the "no" parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        for option in soup.find('select', {'name': 'chapter'}):
+            title = option.string.strip()
+            url = self.READ_URL_TEMPLATE % option['value']
+            self.chapterUrls.append((title, url))
+
+        # Get the URL to the author's page and find the correct story entry to
+        # scrape the metadata
+        author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_no = self.story.getMetadata('storyId')
+        # Ignore first list_box div, it only contains the author information
+        for list_box in soup('div', {'class': 'list_box'})[1:]:
+            url = list_box.find('a', {'class': 'fictitle'})['href']
+            query_data = _get_query_data(url)
+
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if query_data['no'] == story_no:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title_anchor = list_box.find('a', {'class': 'fictitle'})
+        self.story.setMetadata('title', title_anchor.string.strip())
+
+        author_anchor = title_anchor.findNextSibling('a')
+        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
+
+        list_review = list_box.find('div', {'class': 'list_review'})
+        reviews = list_review.a.string.strip().split(' ', 1)[0]
+        self.story.setMetadata('reviews', reviews)
+
+        summary_div = list_box.find('div', {'class': 'list_summary'})
+        if not self.getConfig('keep_summary_html'):
+            summary = ''.join(summary_div(text=True))
+        else:
+            summary = self.utf8FromSoup(author_url, summary_div)
+
+        self.story.setMetadata('description', summary)
+
+        # I'm assuming this to be the category, not sure what else it could be
+        first_listinfo = list_box.find('div', {'class': 'list_info'})
+        self.story.addToList('category', first_listinfo.a.string.strip())
+
+        for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
+            for b_tag in list_info('b'):
+                key = b_tag.string.strip(': ')
+                # Strip colons from the beginning, superfluous spaces and minus
+                # characters from the end, and possibly trailing commas from
+                # the warnings if only one is present
+                value = b_tag.nextSibling.string.strip(': -,')
+
+                if key == 'Genre':
+                    for genre in value.split(', '):
+                        # Ignore the "none" genre
+                        if not genre == 'none':
+                            self.story.addToList('genre', genre)
+
+                elif key == 'Rating':
+                    self.story.setMetadata('rating', value)
+
+                elif key == 'Complete':
+                    self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+                elif key == 'Warning':
+                    for warning in value.split(', '):
+                        # The string here starts with ", " before the actual list
+                        # of values sometimes, so check for an empty warning
+                        # and ignore the "none" warning.
+                        if not warning or warning == 'none':
+                            continue
+
+                        self.story.addToList('warnings', warning)
+
+                elif key == 'Chapters':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Words':
+                    # Apparently only numChapters need to be an integer for
+                    # some strange reason. Remove possible ',' characters as to
+                    # not confuse the codebase down the line
+                    self.story.setMetadata('numWords', value.replace(',', ''))
+
+                elif key == 'Started':
+                    self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
+
+                elif key == 'Updated':
+                    date_string, period = value.rsplit(' ', 1)
+                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
+
+                    # Rather ugly hack to work around Calibre's changing of
+                    # Python's locale setting, causing am/pm to not be properly
+                    # parsed by strptime() when using a non-english locale
+                    if period == 'pm':
+                        date += timedelta(hours=12)
+                    self.story.setMetadata('dateUpdated', date)
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        storytext_div = soup.find('div', {'class': 'storytext'})
+
+        if self.getConfig('strip_text_links'):
+            for anchor in storytext_div('a', {'class': 'FAtxtL'}):
+                navigable_string = BeautifulSoup.NavigableString(anchor.string)
+                anchor.replaceWith(navigable_string)
+
+        return self.utf8FromSoup(url, storytext_div)

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -0,0 +1,336 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BloodTiesFansComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','btf') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'bloodties-fans.com' # XXX
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 4, 2 & 1.
+            # we'll try 1 first.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=561&warning=4
+        # viewstory.php?sid=561&warning=1
+        # viewstory.php?sid=561&warning=2
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fiction/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fiction/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BuffyNFaithNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BuffyNFaithNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.setHeader()
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
+            nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+            #argh, this mangles the ampersands I need on metadata['storyUrl']
+            #will set it this way
+            self.story.setMetadata('storyUrl',nurl,condremoveentities=False)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','bnfnet')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'buffynfaith.net'
+
+    @classmethod
+    def stripURLParameters(cls,url):
+        "Only needs to be overriden if URL contains more than one parameter"
+        ## This adapter needs at least two parameters left on the URL, act and id
+        return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url)
+    
+    def setHeader(self):
+        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
+        
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://buffynfaith.net/fanfictions/index.php?act=ovr&id=1234 http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2"
+
+    def getSiteURLPattern(self):
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2
+        p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\
+            r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        dateformat = "%d %B %Y"
+        url = self.url
+        logger.debug("URL: "+url)
+        
+        #set a cookie to get past adult check
+        if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
+            cookie = cl.Cookie(version=0, name='my_age', value='yes',
+                               port=None, port_specified=False,
+                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
+                               path='/', path_specified=True,
+                               secure=False,
+                               expires=time.time()+10000,
+                               discard=False,
+                               comment=None,
+                               comment_url=None,
+                               rest={'HttpOnly': None},
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
+            self.setHeader()
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        if "ADULT CONTENT WARNING" in data: 
+            raise exceptions.AdultCheckRequired(self.url)
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        #stuff in <head>: description
+        svalue = soup.head.find('meta',attrs={'name':'description'})['content']
+        #self.story.setMetadata('description',svalue)
+        self.setDescription(url,svalue)
+        
+        #useful stuff in rest of doc, all contained in this:
+        doc = soup.body.find('div', id='my_wrapper')
+        
+        #first the site category (more of a genre to me, meh) and title, in this element:
+        mt = doc.find('div',attrs={'class':'maintitle'})
+        self.story.addToList('genre',mt.findAll('a')[1].string)
+        self.story.setMetadata('title',mt.findAll('a')[1].nextSibling[len('&nbsp;&raquo;&nbsp;'):])
+        del mt
+
+        #the actual category, for me, is 'Buffy: The Vampire Slayer'
+        #self.story.addToList('category','Buffy: The Vampire Slayer')
+        #No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini
+        
+        #then a block that sits in a table cell like so:
+        #(contains a lot of metadata)
+        mblock = doc.find('td', align='left', width = '70%').contents
+        while len(mblock) > 0:
+            i = mblock.pop(0)
+            if 'Author:' in i.string:
+                #drop empty space
+                mblock.pop(0)
+                #get author link
+                a = mblock.pop(0)
+                authre = re.escape('./index.php?act=bio&id=')+'(?P<authid>\d+)'
+                m = re.match(authre,a['href'])
+                self.story.setMetadata('author',a.string)
+                self.story.setMetadata('authorId',m.group('authid'))
+                authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('authorId'))
+                self.story.setMetadata('authorUrl',authurl,condremoveentities=False)
+                #drop empty space
+                mblock.pop(0)
+            if 'Rating:' in i.string:
+                self.story.setMetadata('rating',mblock.pop(0).strip())
+            if 'Published:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('datePublished',makeDate(date, dateformat))
+            if 'Last Updated:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('dateUpdated',makeDate(date, dateformat))
+            if 'Genre:' in i.string:
+                genres = mblock.pop(0).strip()
+                genres = genres.split('/')
+                for genre in genres: self.story.addToList('genre',genre)
+            #end ifs
+        #end while
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'ch' } )
+         
+        if select is None:
+           # no selector found, so it's a one-chapter story.
+           #self.chapterUrls.append((self.story.getMetadata('title'),url))
+           self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('storyId'),
+                                            o['value'])
+                title = u"%s" % o
+                title = stripHTML(title)
+                ts = title.split(' ',1)
+                title = ts[0]+'. '+ts[1]
+                self.chapterUrls.append((title,url))
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        ## Go scrape the rest of the metadata from the author's page.
+        data = self._fetchUrl(self.story.getMetadata('authorUrl'))
+        soup = bs.BeautifulSoup(data)
+        #find the story link and its parent div
+        storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
+        storydiv = storya.parent
+        #warnings come under a <spawn> tag. Never seen that before...
+        #appears to just be a line of freeform text, not necessarily a list
+        #optional
+        spawn = storydiv.find('spawn',{'id':'warnings'})
+        if spawn is not None:
+            warns = spawn.nextSibling.strip()
+            self.story.addToList('warnings',warns)
+        #some meta in spans - this should get all, even the ones jammed in a table
+        spans = storydiv.findAll('span')
+        for s in spans:
+            if s.string == 'Ship:':
+                list = s.nextSibling.strip().split()
+                self.story.extendList('ships',list)
+            if s.string == 'Characters:':
+                list = s.nextSibling.strip().split(',')
+                self.story.extendList('characters',list)
+            if s.string == 'Status:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('status',st)
+            if s.string == 'Words:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('numWords',st)
+
+        #reviews - is this worth having?
+        #ffnet adapter gathers it, don't know if anything else does
+        #or if it's ever going to be used!
+        a = storydiv.find('a',{'id':'bold-blue'})
+        if a:
+            revs = a.nextSibling.strip()[1:-1]
+            self.story.setMetadata('reviews',st)
+        else:
+            revs = '0'
+            self.story.setMetadata('reviews',st)
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'fanfiction'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        #remove all the unnecessary bookmark tags
+        [s.extract() for s in div('div',{'class':"tiny_box2"})]
+        
+        #is there a review link?
+        r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$"))
+        if r is not None:
+        #remove the review link and its parent div
+            r.parent.extract()
+        
+        #There might also be a link to the sequel on the last chapter
+        #I'm inclined to keep it in, but the URL needs to be changed from relative to absolute
+        #Shame there isn't proper series metadata available
+        #(I couldn't find it anyway)
+        s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$"))
+        if s is not None:
+            s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:]
+
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -71,7 +72,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -80,11 +81,6 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','cslf') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Castle") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y" # XXX
@@ -94,6 +90,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'castlefans.org' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
 
@@ -122,13 +119,13 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -150,7 +147,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -179,15 +176,16 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
-        
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfic/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -287,6 +285,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -297,7 +296,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','csph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        #self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -70,6 +66,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'chaos.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -91,7 +88,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -119,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -216,6 +213,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -226,7 +224,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -44,30 +45,26 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         self.username = "" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-        
+
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
+
         self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
-        
+
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','chm')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y"
-            
+
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
         return 'www.checkmated.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?story=1234"
 
@@ -82,7 +79,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             return True
         else:
             return False
-        
+
     def performLogin(self, url):
         params = {}
 
@@ -93,14 +90,14 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             params['name'] = self.getConfig("username")
             params['pass'] = self.getConfig("password")
         params['login'] = 'yes'
-        params['submit'] = 'login'	
-	
+        params['submit'] = 'login'
+
         loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
         d = self._fetchUrl(loginUrl,params)
         e = self._fetchUrl(url)
 
         if "Welcome back," not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['name']))
             raise exceptions.FailedToLogin(url,params['name'])
             return False
@@ -117,7 +114,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -135,33 +132,33 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # The actual text that is used to announce you need to be an
         # adult varies from site to site.  Again, print data before
         # the title search to troubleshoot.
-            
+
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
+
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
-        
+
         ## Title
-        a = soup.findAll('span', {'class' : 'storytitle'})
-        self.story.setMetadata('title',a[0].string)
-        
+        a = soup.find('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',stripHTML(a))
+
         # Find authorid and URL from... author url.
-        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        a = a.parent.find('a', href=re.compile(r"authors.php\?name\=\w+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
-			
+
         a = soup.find('select', {'name' : 'chapter'})
         if a == None:
             self.chapterUrls.append((self.story.getMetadata('title'),url))
         else:
             for chapter in a.findAll('option'):
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-			
+
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # eFiction sites don't help us out a lot with their meta data
@@ -174,7 +171,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             except:
                 return ""
 
-				
+
         # website does not keep track of word count, and there is no convenient way to calculate it
 
         summary = soup.find('fieldset')
@@ -182,21 +179,23 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         summary.name='div'
         self.setDescription(url,summary)
 
-		
+
         # <span class="label">Rated:</span> NC-17<br /> etc
         table = soup.findAll('div', {'class' : 'text'})[1]
         for labels in table.findAll('tr'):
             value = labels.findAll('td')[1]
             label = labels.findAll('td')[0]
 
-			
+
             if 'Rating' in stripHTML(label):
                 self.story.setMetadata('rating', stripHTML(value))
 
             if 'Ship' in stripHTML(label):
+                if value.string != "none/none":
+                    self.story.addToList('ships',value.string)
                 for char in value.string.split('/'):
                     if char != 'none':
-                        self.story.addToList('characters',char)		
+                        self.story.addToList('characters',char)
 
             if 'Status' in stripHTML(label):
                 if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
@@ -206,11 +205,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
 
             if 'Published' in stripHTML(label):
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
+
             if 'Updated' in stripHTML(label):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-			
-        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')	
+
+        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')
         for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
             a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
             if a != None:
@@ -220,20 +219,20 @@ class CheckmatedComAdapter(BaseSiteAdapter):
                     if 'genre' in stripHTML(label):
                         for genre in value.findAll('img'):
                             self.story.addToList('genre',genre['title'])
-        
-            
+
+
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
+
         div = soup.find('div', {'id' : 'resizeableText'})
         div.find('div', {'class' : 'storyTools'}).extract()
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -56,14 +56,9 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','dksl')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Buffy the Vampire Slayer")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
+        self.dateformat = "%B %d, %Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
@@ -74,6 +69,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.dark-solace.org','dark-solace.org']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
 
@@ -83,7 +79,8 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
         if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data:
+                or "That password doesn't match the one in our database" in data \
+                or "Registered Users Only" in data:
             return True
         else:
             return False
@@ -97,19 +94,17 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         else:
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
+        params['action'] = 'login'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
-        d = self._fetchUrl(loginUrl, params)
+        d = self._postUrl(loginUrl, params)
     
-        if "User Account Page" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
+        if "Member Account" not in d : #User Account Page
+            logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
         else:
@@ -117,10 +112,20 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
+        
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
-        url = self.url
-        logging.debug("URL: "+url)
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -132,9 +137,30 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         if self.needToLoginCheck(data):
             # need to log in for this one.
-            addurl="&ageconsent=ok"
             self.performLogin(url)
-            data = self._fetchUrl(url+addurl)
+            data = self._fetchUrl(url)
+            
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
@@ -146,34 +172,47 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
+        div = soup.find('div', {'id' : 'pagetitle'})
         
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        aut = div.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',aut['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
         self.story.setMetadata('author',aut.string)
         aut.extract()
+
+        # first a tag in pagetitle is title
+        self.story.setMetadata('title',stripHTML(div.find('a')))
+        div.find('a').extract()
+        # only thing left in div(pagetitle) now should be 'by' and rating.
+        rating = stripHTML(div)
+        if '[' in rating:
+            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
+
+        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
+                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
+            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select', {'name' : 'chapter'})
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
         self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
+
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
-            if a != None:
-                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+        storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                    self.story.getMetadata('storyId')+'($|[^\d])'))
+        # author's story list is paginated if there's a pagelinks div.
+        # Only need to look in it if the story wasn't on the first page.
+        pagelinks = asoup.find('div',{'id':'pagelinks'})
+        if pagelinks and storylink==None:
+            authpageslist = pagelinks.findAll('a',href=re.compile(r'action=storiesby'))
+            for page in authpageslist[1:]: # skip first, already checked above.
+                asoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                            self.story.getMetadata('storyId')+'($|[^\d])'))
+                if storylink:
                     break
 
+        if not storylink:
+            raise exceptions.FailedToDownload("Unable to find story metadata on author's page(s)")
+        
+        metalist = storylink.parent.parent
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
 
@@ -186,7 +225,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
+        labels = metalist.findAll('span', {'class' : 'label'})
         for labelspan in labels:
             label = labelspan.text
             value = labelspan.nextSibling
@@ -194,37 +233,34 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                while value and not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
                 if 'Yes' in value:
@@ -242,34 +278,54 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         try:
             # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            a = metalist.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
             series_name = a.string
             series_url = 'http://'+self.host+'/elysian/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
+            storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                             self.story.getMetadata('storyId')+'($|[^\d])'))
+            if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                storylink = None
+
+            offset = 0
+            # series story list is paginated if there's a pagelinks div.
+            # Only need to look in it if the story wasn't on the first page.
+            pagelinks = seriessoup.find('div',{'id':'pagelinks'})
+            if pagelinks and storylink==None:
+                authpageslist = pagelinks.findAll('a',href=re.compile(r'offset='))
+                for page in authpageslist[1:]: # skip first, already checked above.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                    storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                                     self.story.getMetadata('storyId')+'($|[^\d])'))
+                    if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                        storylink = None
+                    if storylink:
+                        offset = int(page['href'].split('=')[-1]) # offset is last.
                         break
-                    i+=1
+
+            # for reasons I don't understand, searching for story
+            # links by regex wasn't working reliably.  It was missing
+            # the javascript links sometimes.  This is cleaner anyway.
+            for i, div in enumerate(seriessoup.findAll('div', {'class':'title'})):
+                a = div.find('a') # first a is story link.
+                # skip 'report this' and 'TOC' links
+                if a == storylink:
+                    self.setSeries(series_name, 1+i+offset)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
             
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        except Exception, e:
+            print("Series parsing failed: %s"%e)
             
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -64,6 +65,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.destinysgateway.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -86,7 +88,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -106,7 +108,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -129,7 +131,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -217,6 +219,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -227,7 +230,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_dokugacom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,14 @@ class DokugaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+
+        # www.dokuga.com has two 'sections', shown in URL as
+        # 'fanfiction' and 'spark' that change how things should be
+        # handled.
+        # http://www.dokuga.com/fanfiction/story/7528/1
+        # http://www.dokuga.com/spark/story/7299/1
+        self.section=self.parsedUrl.path.split('/',)[1]
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
@@ -56,14 +63,9 @@ class DokugaComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','dkg')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","InuYasha")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        if 'fanfiction' in self.story.getMetadata('section'):
+        if 'fanfiction' in self.section:
             self.dateformat = "%d %b %Y"
         else:
             self.dateformat = "%m-%d-%y"
@@ -73,12 +75,51 @@ class DokugaComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.dokuga.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
 
     def getSiteURLPattern(self):
         return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'The author has disabled anonymous viewing for this story.' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url,soup):
+        params = {}
+
+        if self.password:
+            params['username'] = self.username
+            params['passwd'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['passwd'] = self.getConfig("password")
+        params['Submit'] = 'Submit'
+
+        # copy all hidden input tags to pick up appropriate tokens.
+        for tag in soup.findAll('input',{'type':'hidden'}):
+            params[tag['name']] = tag['value']
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['username']))
+    
+        d = self._postUrl(loginUrl, params)
+
+        if "Your session has expired. Please log in again." in d:
+            d = self._postUrl(loginUrl, params)
+
+        if "Logout" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['username']))
+            raise exceptions.FailedToLogin(url,params['username'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -86,7 +127,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -95,12 +136,18 @@ class DokugaComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+                
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url,soup)
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
@@ -117,22 +164,22 @@ class DokugaComAdapter(BaseSiteAdapter):
         aut.extract()
         
         a = a.string[:(len(a.string)-4)]
-        self.story.setMetadata('title',a)
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters:
         chapters = soup.find('select').findAll('option')
         if len(chapters)==1:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/1'))
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1'))
         else:
             for chapter in chapters:
                 # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
         
         asoup = bs.BeautifulSoup(self._fetchUrl(alink))
         
-        if 'fanfiction' in self.story.getMetadata('section'):
+        if 'fanfiction' in self.section:
             asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
         
             #grab the rest of the metadata from the author's page
@@ -158,7 +205,6 @@ class DokugaComAdapter(BaseSiteAdapter):
             
             a=div.text.split('Category: ')
             if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
-            self.story.addToList('category', 'Fanfiction')
         
             a=div.text.split('Created: ')
             if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
@@ -216,14 +262,10 @@ class DokugaComAdapter(BaseSiteAdapter):
             a=div.text.split('Words ')
             if len(a)==2: self.story.setMetadata('numWords', a[1])
             
-        
-
-
-            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DotMoonNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DotMoonNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL. www.dotmoon.net/library_view.php?storyid=3
+        self._setURL('http://' + self.getSiteDomain() + '/library_view.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dotm')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dotmoon.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/library_view.php?storyid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'You must be logged in to read adult-rated stories' in data \
+                or 'Password incorrect' in data \
+                or "That username does not exist" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['user'] = self.username
+            params['passwrd'] = self.password
+        else:
+            params['user'] = self.getConfig("username")
+            params['passwrd'] = self.getConfig("password")
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/board/index.php'
+        
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['user']))
+    
+        d = self._fetchUrl(loginUrl+'?action=login2&user='+params['user']+'&passwrd='+params['passwrd'])
+        d = self._fetchUrl(loginUrl)
+    
+        if "Show unread posts since last visit" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['user']))
+            raise exceptions.FailedToLogin(url,params['user'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Invalid story ID" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Invalid story ID.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        body=soup.findAll('body')[1]
+        body.find('table').extract()
+        
+        ## Title
+        a = body.find('b')
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
+        a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters: 'library_storyview.php?chapterid=3
+        chapters=body.findAll('a', href=re.compile(r"library_storyview.php\?chapterid=\d+$"))
+        if len(chapters)==0:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.")
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # other tags
+
+        labels = body.find('table', {'width':'390'}).findAll('td')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if label != None:
+                if 'Fandom' in label:
+                    self.story.addToList('category',value.string)
+                
+                if 'Setting' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Genre' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Style' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Rating' in label:
+                    self.story.addToList('rating',value.string)
+                
+                if 'Created' in label:
+                    self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+                if 'Updated' in label:
+                    self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+        
+                if 'Status' in label:
+                    if 'Completed' in value.string:
+                        self.story.setMetadata('status', 'Completed')
+                    else:
+                        self.story.setMetadata('status', 'In-Progress')
+                        
+        table=body.findAll('table', {'width':'400'})[1].find('td')
+        self.setDescription(url,stripHTML(table).split('Summary: ')[1])
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('blockquote')
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,13 +56,6 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','dcagn')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-        self.story.addToList("characters","Ginny Weasley")
-        self.story.addToList("characters","Draco Malfoy")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y"
@@ -71,6 +65,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.dracoandginny.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -99,13 +94,13 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -127,7 +122,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -152,7 +147,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -175,7 +170,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -279,6 +274,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -289,12 +285,15 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'class' : 'listbox'})
+        
+        if None == div:
+            div = soup.find('div', {'id' : 'story'})
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','drmn')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d %B %Y"
@@ -69,6 +65,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'dramione.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -97,13 +94,13 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -125,7 +122,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -157,7 +154,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -165,6 +162,20 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
+        # Use banner as cover if found
+        coverurl = ''
+        img = soup.find('img',{'class':'banner'})
+        if img:
+            coverurl = img['src']
+            #print "Cover: "+coverurl
+        a = soup.find(text="This story has a banner; click to view.")
+        if a:
+            #print "A: "+ ', '.join("(%s, %s)" %tup for tup in a.parent.attrs)
+            coverurl = a.parent['href']
+            #print "Cover: "+coverurl
+        if coverurl:
+            self.setCoverImage(url,coverurl)
+
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
@@ -183,6 +194,22 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         for warning in warnings:
             self.story.addToList('warnings',warning.string)
 
+        themes=soup.findAll('a', {'class' : "tag-3"})
+        for theme in themes:
+            self.story.addToList('themes',theme.string)
+
+        hermiones=soup.findAll('a', {'class' : "tag-4"})
+        for hermione in hermiones:
+            self.story.addToList('hermiones',hermione.string)
+
+        dracos=soup.findAll('a', {'class' : "tag-5"})
+        for draco in dracos:
+            self.story.addToList('dracos',draco.string)
+
+        timelines=soup.findAll('a', {'class' : "tag-6"})
+        for timeline in timelines:
+            self.story.addToList('timeline',timeline.string)
+
         # utility method
         def defaultGetattr(d,k):
             try:
@@ -211,6 +238,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -248,6 +278,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -255,10 +286,18 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             # I find it hard to care if the series parsing fails
             pass
             
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('h2',{'id':'pagetitle'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return EfictionEstelielDeAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class EfictionEstelielDeAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','eesd')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'efiction.esteliel.de'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        list = soup.find('div', {'class':'listbox'})
+        labelspan=list.find('span',{'class':'label'})
+        value = labelspan.nextSibling
+        label = labelspan.string
+        genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+        
+        labels = list.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while 'Rating' not in str(value):
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Words' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Category' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            if list.find('a', href=re.compile(r"series.php")) != None:
+                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
+                    # Find Series name from series URL.
+                    series_url = 'http://'+self.host+'/'+series['href']
+                    # use BeautifulSoup HTML parser to make everything easier to find.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+                    i=1
+                    for a in storyas:
+                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                            name=seriessoup.find('div', {'id' : 'pagetitle'})
+                            name.find('a').extract()
+                            self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
+                            i=0
+                            break
+                        i+=1
+                    if i == 0:
+                        break
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -0,0 +1,314 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return EFPFanFicNet
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class EFPFanFicNet(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','efp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.efpfanfic.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Fai il login e leggi la storia!' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Invia'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?sid='+self.story.getMetadata('storyId')
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if '<a class="menu" href="newaccount.php">' in d : # register for new account link
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'sid' } )
+    	 
+        if select is None:
+    	   # no selector found, so it's a one-chapter story.
+    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option', {'value' : re.compile(r'viewstory')})
+            for o in allOptions:
+                url = u'http://%s/%s' % ( self.getSiteDomain(),
+                                          o['value'])
+                # just in case there's tags, like <i> in chapter titles.
+                title = stripHTML(o)
+                self.chapterUrls.append((title,url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','Italian')
+        
+        # normalize story URL to first chapter if later chapter URL was given:
+        url = self.chapterUrls[0][1].replace('&i=1','')
+        logger.debug("Normalizing to URL: "+url)
+        self._setURL(url)
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        storya = None
+        authsoup = None
+        storyblock = None
+        authurl = self.story.getMetadata('authorUrl')
+        
+        ## author can have more than one page of stories.
+        while storyblock == None:
+
+            # no storya, but do have authsoup--we're looping on author pages.
+            if authsoup != None:
+                # last author link with offset should be the 'next' link.
+                authurl = u'http://%s/%s' % ( self.getSiteDomain(),
+                                              authsoup.findAll('a',href=re.compile(r'viewuser\.php\?uid=\d+&catid=&offset='))[-1]['href'] )
+            
+            # Need author page for most of the metadata.
+            logger.debug("fetching author page: (%s)"%authurl)
+            authsoup = bs.BeautifulSoup(self._fetchUrl(authurl))
+            #print("authsoup:%s"%authsoup)
+
+            storyas = authsoup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+r'&i=1$'))
+            for storya in storyas:
+                #print("======storya:%s"%storya)
+                storyblock = storya.findParent('div',{'class':'storybloc'})
+                #print("======storyblock:%s"%storyblock)
+                if storyblock != None:
+                    continue
+
+        self.setDescription(url,storyblock.find('div', {'class':'introbloc'}))
+
+        noteblock = storyblock.find('div', {'class':'notebloc'})
+        #print("%s"%noteblock)
+        notetext = ("%s" % noteblock).replace("<br />"," |")
+        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />
+        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />
+        # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>
+
+        cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        for item in notetext.split("|"):
+            if ":" in item:
+                (label,value) = item.split(":")
+                label=label.strip()
+                value=value.strip()
+            else:
+                label=value=item.strip()
+
+            if 'Pubblicata' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Aggiornata' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            if label == "Completa":
+                self.story.setMetadata('status', 'Completed')
+            
+            if label == "In corso":
+                self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Personaggi' in label:
+                for val in value.split(","):
+                    self.story.addToList('characters',val)
+                    
+            if 'Genere' in label:
+                for val in value.split(","):
+                    self.story.addToList('genre',val)
+                    
+            if 'Coppie' in label:
+                for val in value.split(","):
+                    self.story.addToList('ships',val)
+
+            if 'Avvertimenti' in label:
+                for val in value.split(","):
+                    if val != "None":
+                        self.story.addToList('warnings',val)
+
+            # 'extra' metadata for this adapter:
+                    
+            if 'Tipo di coppia' in label:
+                for val in value.split(","):
+                    self.story.addToList('type',val)
+                    
+            if 'Note' in label:
+                for val in value.split(","):
+                    if val != "None":
+                        self.story.addToList('notes',val)
+                    
+            if 'Contesto' in label:
+                self.story.setMetadata('context', value)
+
+            ## Note--efp doesn't provide word count.
+                
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?ssid=\d+&i=1"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'class' : 'storia'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        # remove any header and 'o:p' tags.
+        for tag in div.findAll("head") + div.findAll("o:p"):
+            tag.extract()
+
+        # change any html and body tags to div.
+        for tag in div.findAll("html") + div.findAll("body"):
+            tag.name='div'
+
+        # remove extra bogus doctype.
+        #<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+        return re.sub(r"<!DOCTYPE[^>]+>","",self.utf8FromSoup(url,div))

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','essph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y"
@@ -70,6 +66,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'erosnsappho.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -91,7 +88,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -111,7 +108,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -135,7 +132,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -234,6 +231,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -244,7 +242,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_fanfichu.py

@@ -0,0 +1,185 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanficHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class FanficHuAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'ffh'
+    SITE_DOMAIN = 'fanfic.hu'
+    SITE_LANGUAGE = 'Hungarian'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/merengo/'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%s'
+
+    DATE_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanficHuAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&i=1')
+
+        if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        chapter_options = soup.find('form', action='viewstory.php').select('option')
+        # Remove redundant "Fejezetek" option
+        chapter_options.pop(0)
+
+        # If there is still more than one entry remove chapter overview entry
+        if len(chapter_options) > 1:
+            chapter_options.pop(0)
+
+        for option in chapter_options:
+            url = urlparse.urljoin(self.url, option['value'])
+            self.chapterUrls.append((option.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_id = self.story.getMetadata('storyId')
+        for table in soup('table', {'class': 'mainnav'}):
+            title_anchor = table.find('span', {'class': 'storytitle'}).a
+            href = title_anchor['href']
+            if href.startswith('javascript:'):
+                href = href.rsplit(' ', 1)[1].strip("'")
+            query_data = _get_query_data(href)
+
+            if query_data['sid'] == story_id:
+                break
+        else:
+            # This should never happen, the story must be found on the author's
+            # page.
+            raise exceptions.FailedToDownload(self.url)
+
+        self.story.setMetadata('title', title_anchor.string)
+
+        rows = table('tr')
+
+        anchors = rows[0].div('a')
+        author_anchor = anchors[1]
+        query_data = _get_query_data(author_anchor['href'])
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.BASE_URL, author_anchor['href']))
+        self.story.setMetadata('reviews', anchors[3].string)
+
+        if self.getConfig('keep_summary_html'):
+            self.story.setMetadata('description', self.utf8FromSoup(author_url, rows[1].td))
+        else:
+            self.story.setMetadata('description', ''.join(rows[1].td(text=True)))
+
+        for row in rows[3:]:
+            index = 0
+            cells = row('td')
+
+            while index < len(cells):
+                cell = cells[index]
+                key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
+                try:
+                    value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
+                except AttributeError:
+                    value = None
+
+                if key == 'Kategória':
+                    for anchor in cells[index+1]('a'):
+                        self.story.addToList('category', anchor.string)
+
+                elif key == 'Szereplõk':
+                    if cells[index+1].string:
+                        for name in cells[index+1].string.split(', '):
+                            self.story.addToList('character', name)
+
+                elif key == 'Korhatár':
+                    if value != 'nem korhatáros':
+                        self.story.setMetadata('rating', value)
+
+                elif key == 'Figyelmeztetések':
+                    for b_tag in cells[index+1]('b'):
+                        self.story.addToList('warnings', b_tag.string)
+
+                elif key == 'Jellemzõk':
+                    for genre in cells[index+1].string.split(', '):
+                        self.story.addToList('genre', genre)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Megjelenés':
+                    self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Frissítés':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Szavak':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                index += 2
+
+        if self.story.getMetadata('rating') == '18':
+            if not (self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        story_cell = soup.find('form', action='viewstory.php').parent.parent
+
+        for div in story_cell('div'):
+            div.extract()
+
+        return self.utf8FromSoup(url, story_cell)

fanficdownloader/adapters/adapter_fanfictioncsodaidokhu.py

@@ -0,0 +1,218 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanfictionCsodaidokHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FanfictionCsodaidokHuAdapter(BaseSiteAdapter):
+    _SITE_DOMAIN = 'fanfiction.csodaidok.hu'
+    _BASE_URL = 'http://' + _SITE_DOMAIN + '/'
+    _VIEW_STORY_URL_TEMPLATE = _BASE_URL + 'viewstory.php?sid=%s'
+    _VIEW_CHAPTER_URL_TEMPLATE = _VIEW_STORY_URL_TEMPLATE + '&chapter=%s'
+
+    _STORY_DOES_NOT_EXIST_PAGE_TITLE = 'Cím:  Szerző:'
+    _DATE_FORMAT = '%Y.%m.%d'
+    _SITE_LANGUAGE = 'Hungarian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self._VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self._SITE_DOMAIN)
+        self.story.setMetadata('language', self._SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanfictionCsodaidokHuAdapter._SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls._VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self._VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&chapter=1')
+
+        element = soup.find('div', id='pagetitle')
+        page_title = ''.join(element(text=True)).encode(_SOURCE_CODE_ENCODING)
+        if page_title == self._STORY_DOES_NOT_EXIST_PAGE_TITLE:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        author_url = urlparse.urljoin(self.url, element.a['href'])
+
+        story_id = self.story.getMetadata('storyId')
+        element = soup.find('select', {'name': 'chapter'})
+        if element:
+            for option in element('option'):
+                title = option.string
+                url = self._VIEW_CHAPTER_URL_TEMPLATE % (story_id, option['value'])
+                self.chapterUrls.append((title, url))
+
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+
+        for listbox_div in soup('div', {'class': lambda klass: klass and 'listbox' in klass}):
+            a = listbox_div.div.a
+            if not a['href'].startswith('viewstory.php?sid='):
+                continue
+
+            query_data = _get_query_data(a['href'])
+            if query_data['sid'] == story_id:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = ''.join(a(text=True))
+        self.story.setMetadata('title', title)
+        if not self.chapterUrls:
+            self.chapterUrls.append((title, self.url))
+
+        element = a.findNextSibling('a')
+        self.story.setMetadata('author', element.string)
+        query_data = _get_query_data(element['href'])
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', author_url)
+
+        element = element.findNextSibling('span')
+        rating = element.nextSibling.strip(' [')
+
+        if rating.encode(_SOURCE_CODE_ENCODING) != 'Korhatár nélkül':
+            self.story.setMetadata('rating', rating)
+
+        if rating == '18':
+            raise exceptions.AdultCheckRequired(self.url)
+
+        element = element.findNextSiblings('a')[1]
+        self.story.setMetadata('reviews', element.string)
+
+        sections = listbox_div('div', {'class': lambda klass: klass and klass in ['content', 'tail']})
+        for section in sections:
+            for element in section('span', {'class': 'classification'}):
+                key = element.string.encode(_SOURCE_CODE_ENCODING).strip(' :')
+                try:
+                    value = element.nextSibling.string.encode(_SOURCE_CODE_ENCODING).strip()
+                except AttributeError:
+                    value = None
+
+                if key == 'Tartalom':
+                    contents = []
+                    keep_summary_html = self.getConfig('keep_summary_html')
+
+                    for sibling in _yield_next_siblings(element):
+                        if isinstance(sibling, BeautifulSoup.Tag):
+                            if sibling.name == 'span' and sibling.get('class', None) == 'classification':
+                                break
+
+                            if keep_summary_html:
+                                contents.append(self.utf8FromSoup(author_url, sibling))
+                            else:
+                                contents.append(''.join(sibling(text=True)))
+                        else:
+                            contents.append(sibling)
+                    self.story.setMetadata('description', ''.join(contents))
+
+                elif key == 'Kategória':
+                    for sibling in element.findNextSiblings(['a', 'span']):
+                        if sibling.name == 'span':
+                            break
+
+                        self.story.addToList('category', sibling.string)
+
+                elif key == 'Szereplők':
+                    for name in value.split(', '):
+                        self.story.addToList('characters', name)
+
+                elif key == 'Műfaj':
+                    if value != 'Nincs':
+                        self.story.setMetadata('genre', value)
+
+                elif key == 'Figyelmeztetés':
+                    if value != 'Nincs':
+                        for warning in value.split(', '):
+                            self.story.addToList('warnings', warning)
+
+                elif key == 'Kihívás':
+                    if value != 'Nincs':
+                        self.story.setMetadata('challenge', value)
+
+                elif key == 'Sorozat':
+                    if value != 'Nincs':
+                        self.story.setMetadata('series', value)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                elif key == 'Szavak száma':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Feltöltve':
+                    self.story.setMetadata('datePublished', makeDate(value, self._DATE_FORMAT))
+
+                elif key == 'Frissítve':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self._DATE_FORMAT))
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        contents = []
+
+        notes_div = soup.find('div', id='notes')
+        if notes_div:
+            contents.append(self.utf8FromSoup(url, notes_div))
+            story_div = notes_div.findNextSibling('div')
+        else:
+            element = soup.find('div', {'class': 'jumpmenu'})
+            story_div = element.findNextSibling('div')
+
+        contents.append(self.utf8FromSoup(url, story_div.span))
+        return ''.join(contents)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -16,7 +16,9 @@
 #
 
 import time
+from datetime import datetime
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 from urllib import unquote_plus
@@ -42,7 +44,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
 
         # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
+        self._setURL("https://"+self.getSiteDomain()\
                          +"/s/"+self.story.getMetadata('storyId')+"/1/")
 
         # ffnet update emails have the latest chapter URL.
@@ -51,9 +53,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # chapter list doesn't get the latest.  So save and use the
         # original URL given to pull chapter list & metadata.
         self.origurl = url
-        if "http://m." in self.origurl:
+        if "https://m." in self.origurl:
             ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("http://m.","http://www.")
+            self.origurl = self.origurl.replace("https://m.","https://www.")
+
+        self.opener.addheaders.append(('Referer',self.origurl))
 
     @staticmethod
     def getSiteDomain():
@@ -63,11 +67,19 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.fanfiction.net','m.fanfiction.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
+        return "https://www.fanfiction.net/s/1234/1/ https://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
+
+    def _fetchUrl(self,url):
+        time.sleep(1.0) ## ffnet(and, I assume, fpcom) tends to fail
+                        ## more if hit too fast.  This is in
+                        ## additional to what ever the
+                        ## slow_down_sleep_time setting is.
+        return BaseSiteAdapter._fetchUrl(self,url)
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -75,12 +87,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # metadata and chapter list
 
         url = self.origurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
-            #print("\n===================\n%s\n===================\n"%data)
+            #logger.debug("\n===================\n%s\n===================\n"%data)
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -95,32 +107,35 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
+        if self.getConfig('check_next_chapter'):
             try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
+                ## ffnet used to have a tendency to send out update
+                ## notices in email before all their servers were
+                ## showing the update on the first chapter.  It
+                ## generates another server request and doesn't seem
+                ## to be needed lately, so now default it to off.
+                try:
+                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+                # get chapter part of url.
+                except:
+                    chapcount = 1
+                chapter = url.split('/',)[5]
+                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                                self.story.getMetadata('storyId'),
+                                                chapcount+1)
+                logger.debug('=Trying newer chapter: %s' % tryurl)
+                newdata = self._fetchUrl(tryurl)
+                if "not found. Please check to see you are not using an outdated url." \
+                        not in newdata:
+                    logger.debug('=======Found newer chapter: %s' % tryurl)
+                    soup = bs.BeautifulSoup(newdata)
             except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                           self.story.getMetadata('storyId'),
-                                           chapcount+1)
-            print('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                print('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
+                pass
+            
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
 
         ## Pull some additional data from html.
@@ -131,16 +146,30 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         ## For 1, use the second link.
         ## For 2, fetch the crossover page and pull the two categories from there.
 
-        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
+        #print("xcontrast_txt a:%s"%categories)
         if len(categories) > 1:
-            self.story.addToList('category',stripHTML(categories[-1]))
+            # Strangely, the ones with *two* links are the
+            # non-crossover categories.  Each is in a category itself
+            # of Book, Movie, etc.
+            self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
-            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
+            caturl = "https://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
-            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
+            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/.+?/\d+/")):
                 self.story.addToList('category',stripHTML(a))
+            else:
+                # Fall back.  I ran across a story with a Crossver
+                # category link to a broken page once.
+                # http://www.fanfiction.net/s/2622060/1/
+                # Naruto + Harry Potter Crossover
+                logger.info("Fall back category collection")
+                for c in stripHTML(categories[0]).replace(" Crossover","").split(' + '):
+                    self.story.addToList('category',c)
+                
+                
             
-        a = soup.find('a', href='http://www.fictionratings.com/')
+        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
         rating = a.string
         if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
             rating = rating[8:]
@@ -149,7 +178,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find(id="gui_table1i")
+        gui_table1i = soup.find('div',{'id':'content_wrapper_inner'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -157,12 +186,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if summarydiv:
             self.setDescription(url,stripHTML(summarydiv))
             
-        
-        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
+
+        grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
+        # for b in grayspan.findAll('button'):
+        #     b.extract()
+        metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
+        #logger.debug("metatext:(%s)"%metatext)
         metalist = metatext.split(" - ")
-        #print("metatext:(%s)"%metalist)
+        #logger.debug("metalist:(%s)"%metalist)
 
         # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+        # Rated: Fiction T - English - Adventure/Sci-Fi - Naruto U. - Chapters: 22 - Words: 114,414 - Reviews: 395 - Favs: 779 - Follows: 835 - Updated: 03-21-13 - Published: 04-28-12 - id: 8067258 
 
         # rating is obtained above more robustly.
         if metalist[0].startswith('Rated:'):
@@ -176,42 +210,60 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
         goodgenres=True
         for g in genrelist:
-            print("g:(%s)"%g)
+            #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                print("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
             metalist=metalist[1:]
 
+        # Updated: <span data-xutime='1368059198'>5/8</span> - Published: <span data-xutime='1278984264'>7/12/2010</span>
+        # Published: <span data-xutime='1384358726'>8m ago</span>
+        dates = soup.findAll('span',{'data-xutime':re.compile(r'^\d+$')})
+        if len(dates) > 1 :
+            # updated get set to the same as published upstream if not found.
+            self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
+        self.story.setMetadata('datePublished',datetime.fromtimestamp(float(dates[-1]['data-xutime'])))
+            
         donechars = False
         while len(metalist) > 0:
-            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Favs:') or metalist[0].startswith('Follows:'):
+            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Updated:') or metalist[0].startswith('Published:'):
                 pass
-            elif  metalist[0].startswith('Updated'):
-                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
-            elif  metalist[0].startswith('Published'):
-                self.story.setMetadata('datePublished',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
+            elif  metalist[0].startswith('Reviews'):
+                self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Favs:'):
+                self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Follows:'):
+                self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
             elif  metalist[0].startswith('Words'):
                 self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
             elif not donechars:
-                self.story.extendList('characters',metalist[0].split('&'))
+                # with 'pairing' support, pairings are bracketed w/o comma after
+                # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
+                self.story.extendList('characters',metalist[0].replace('[','').replace(']',',').split(','))
+
+                l = metalist[0]
+                while '[' in l:
+                    self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
+                    l = l[l.index(']')+1:]
+
                 donechars = True
             metalist=metalist[1:]
 
-        # next might be characters, otherwise Reviews, Updated, Published, Words
-        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
-        #     self.story.extendList('characters',metalist[0].split('&'))
-        
         if 'Status: Complete' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        if self.getConfig('include_images'):
+        # Try the larger image first.
+        try:
+            img = soup.find('img',{'class':'lazy cimage'})
+            self.setCoverImage(url,img['data-original'])
+        except:
             img = soup.find('img',{'class':'cimage'})
             if img:
-                self.story.addImgUrl(self,url,img['src'],self._fetchUrlRaw,cover=True)
+                self.setCoverImage(url,img['src'])
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapter' } )
@@ -222,9 +274,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             allOptions = select.findAll('option')
             for o in allOptions:
-                url = u'http://%s/s/%s/%s/' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            o['value'])
+                url = u'https://%s/s/%s/%s/' % ( self.getSiteDomain(),
+                                                 self.story.getMetadata('storyId'),
+                                                 o['value'])
                 # just in case there's tags, like <i> in chapter titles.
                 title = u"%s" % o
                 title = re.sub(r'<[^>]+>','',title)
@@ -235,19 +287,26 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(4.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
 
+        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
+        
         # some ancient stories have body tags inside them that cause
         # soup parsing to discard the content.  For story text we
         # don't care about anything before "<div class='storytextp"
         # (there's a space after storytextp, so no close quote(')) and
         # this kills any body tags.
-        data = data[data.index("<div class='storytextp"):]
+        divstr = "<div role='main'"
+        if divstr not in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        else:
+            data = data[data.index(divstr):]
         data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
         
         soup = bs.BeautifulSoup(data)
@@ -260,7 +319,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         div = soup.find('div', {'id' : 'storytextp'})
         
         if None == div:
-            logging.debug('div id=storytextp not found.  data:%s'%data)
+            logger.debug('div id=storytextp not found.  data:%s'%data)
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -25,6 +26,7 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
+
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
@@ -48,7 +50,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
@@ -65,11 +67,12 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.fanfiktion.de'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?"
         
         ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -94,12 +97,12 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         params['submit'] = 'Login...'
 
         loginUrl = 'https://ssl.fanfiktion.de/'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['nickname']))
         d = self._postUrl(loginUrl,params)
     
         if "Login erfolgreich" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['nickname']))
             raise exceptions.FailedToLogin(url,params['nickname'])
             return False
@@ -110,7 +113,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -135,15 +138,15 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"/"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        head = soup.find('div', {'style' : 'width:85%;float:left;'})
+        head = soup.find('div', {'class' : 'story-metadata-left-top'})
         a = head.find('a')
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
 
         # Find the chapters:
         for chapter in soup.find('select').findAll('option'):
@@ -170,20 +173,20 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         tr=asoup.findAll('tr')
         for i in range(1,len(tr)):
             a = tr[i].find('a')
-            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
+            if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
                 break
         self.setDescription(url,a['onmouseover'].split("', '")[1])
         
         td = tr[i].findAll('td')
-        self.story.addToList('category',td[1].string)
-        self.story.setMetadata('rating', td[4].string)
-        self.story.setMetadata('numWords', td[5].string)
+        self.story.addToList('category',stripHTML(td[1]))
+        self.story.setMetadata('rating', stripHTML(td[4]))
+        self.story.setMetadata('numWords', stripHTML(td[5]))
  
             
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffde has "floodlock" protection
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -18,6 +18,7 @@
 import time
 import datetime
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 from .. import translit
@@ -33,6 +34,8 @@ def getClass():
     return FicBookNetAdapter
 
 
+logger = logging.getLogger(__name__)
+
 class FicBookNetAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -49,7 +52,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
@@ -66,8 +69,9 @@ class FicBookNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.ficbook.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/readfic/12345"
+        return "http://"+self.getSiteDomain()+"/readfic/12345 http://"+self.getSiteDomain()+"/readfic/93626/246417#part_content"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
@@ -75,7 +79,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
         url=self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
         try:
             data = self._fetchUrl(url)
         except urllib2.HTTPError, e:
@@ -95,14 +99,14 @@ class FicBookNetAdapter(BaseSiteAdapter):
         ## Title
         a = soup.find('h1')
         self.story.setMetadata('title',stripHTML(a))
-        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
+        logger.debug("Title: (%s)"%self.story.getMetadata('title'))
         
         # Find authorid and URL from... author url.
         a = table.find('a')
         self.story.setMetadata('authorId',a.text) # Author's name is unique
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.text)
-        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
+        logger.debug("Author: (%s)"%self.story.getMetadata('author'))
 
         # Find the chapters:
         chapters = soup.find('div', {'class' : 'part_list'})
@@ -123,7 +127,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
             pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
             update=pubdate
 
-        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
 
         if not ',' in pubdate:
             pubdate=datetime.date.today().strftime(self.dateformat)
@@ -201,13 +205,13 @@ class FicBookNetAdapter(BaseSiteAdapter):
                 break
 				
         summary=soup.find('span', {'class' : 'urlize'})
-        self.setDescription(url,summary.text)
+        self.setDescription(url,summary)
         #self.story.setMetadata('description', summary.text)
             
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,7 +38,6 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query correct
@@ -45,7 +45,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if m:
             self.story.setMetadata('authorId',m.group('auth'))
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL(url)
         else:
@@ -57,6 +57,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.fictionalley.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/authors/drt/DA.html http://"+self.getSiteDomain()+"/authors/drt/JOTP01a.html"
 
@@ -69,7 +70,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if self.is_adult or self.getConfig("is_adult"):
             params={'iamold':'Yes',
                     'action':'ageanswer'}
-            logging.info("Attempting to get cookie for %s" % url)
+            logger.info("Attempting to get cookie for %s" % url)
             ## posting on list doesn't work, but doesn't hurt, either.
             data = self._postUrl(url,params)
         else:
@@ -80,7 +81,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
         ## could be either chapter list page or one-shot text page.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._postFetchWithIAmOld(url)
@@ -108,9 +109,9 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             storya = soup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
             self._setURL(storya['href'])
             url=self.url
-            logging.debug("Normalizing to URL: "+url)
+            logger.debug("Normalizing to URL: "+url)
             ## title's right there...
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
             data = self._fetchUrl(url)
             soup = bs.BeautifulSoup(data)
             chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
@@ -120,7 +121,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             ## same way.
             chapsoup = bs.BeautifulSoup(chapterdata)
             storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
             del chapsoup
 
         del chapterdata
@@ -194,7 +195,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
 	# find <!-- headerend --> & <!-- footerstart --> and
@@ -223,7 +224,16 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             
         if not data or not text:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
+        # not sure how, but we can get html, etc tags still in some
+        # stories.  That breaks later updates because it confuses
+        # epubutils.py
+        for tag in text.findAll('head'):
+            tag.extract()
+        
+        for tag in text.findAll('body') + text.findAll('html'):
+            tag.name = 'div'
+            
         return self.utf8FromSoup(url,text)
 
 def getClass():

fanficdownloader/adapters/adapter_fictionmaniatv.py

@@ -0,0 +1,178 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+from ..BeautifulSoup import NavigableString
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return FictionManiaTVAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FictionManiaTVAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'fmt'
+    SITE_DOMAIN = 'fictionmania.tv'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/stories/'
+    READ_TEXT_STORY_URL_TEMPLATE = BASE_URL + 'readtextstory.html?storyID=%s'
+    DETAILS_URL_TEMPLATE = BASE_URL + 'details.html?storyID=%s'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    ALTERNATIVE_DATETIME_FORMAT = '%m/%d/%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['storyID'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.READ_TEXT_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+        # Always single chapters, probably should use the Anthology feature to
+        # merge chapters of a story
+        self.story.setMetadata('numChapters', 1)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FictionManiaTVAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_TEXT_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL) + '(readtextstory|details)\.html\?storyID=\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        url = self.DETAILS_URL_TEMPLATE % self.story.getMetadata('storyId')
+        soup = self._customized_fetch_url(url)
+
+        keep_summary_html = self.getConfig('keep_summary_html')
+        for row in soup.find('table')('tr'):
+            cells = row('td')
+            key = cells[0].b.string.strip(':')
+            try:
+                value = cells[1].string
+            except AttributeError:
+                value = None
+
+            if key == 'Story Name-Title':
+                self.story.setMetadata('title', value)
+                self.chapterUrls.append((value, self.url))
+
+            elif key == 'File Name':
+                self.story.setMetadata('fileName', value)
+
+            elif key == 'File Size':
+                self.story.setMetadata('fileSize', value)
+
+            elif key == 'Author':
+                element = cells[1].a
+                self.story.setMetadata('author', element.string)
+                query_data = _get_query_data(element['href'])
+                self.story.setMetadata('authorId', query_data['word'])
+                self.story.setMetadata('authorUrl', urlparse.urljoin(url, element['href']))
+
+            elif key == 'Date Added':
+                try:
+                    date = makeDate(value, self.DATETIME_FORMAT)
+                except ValueError:
+                    date = makeDate(value, self.ALTERNATIVE_DATETIME_FORMAT)
+                self.story.setMetadata('datePublished', date)
+
+            elif key == 'Old Name':
+                self.story.setMetadata('oldName', value)
+
+            elif key == 'New Name':
+                self.story.setMetadata('newName', value)
+
+            elif key == 'Other Key Names':
+                for name in value.split(', '):
+                    self.story.addToList('characters', name)
+
+            # I have no clue how the rating system works, if you are reading
+            # transgender fanfiction, you are probably an adult.
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Complete':
+                self.story.setMetadata('status', 'Complete' if value == 'Complete' else 'In-Progress')
+
+            elif key == 'Categories':
+                for element in cells[1]('a'):
+                    self.story.addToList('category', element.string)
+
+            elif key == 'Key Words':
+                for element in cells[1]('a'):
+                    self.story.addToList('keyWords', element.string)
+
+            elif key == 'Main Characters Age':
+                element = cells[1].a
+                self.story.setMetadata('mainCharactersAge', element.string)
+
+            elif key == 'Synopsis':
+                element = cells[1]
+
+                # Replace td with div to avoid possible strange formatting in
+                # the ebook later on
+                element.name = 'div'
+
+                if keep_summary_html:
+                    self.story.setMetadata('description', unicode(element))
+                else:
+                    self.story.setMetadata('description', ''.join(element(text=True)))
+
+            elif key == 'Reads':
+                self.story.setMetadata('readings', value)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        element = soup.find('pre')
+        element.name = 'div'
+
+        # The story's content is contained in a <pre> tag, probably taken 1:1
+        # from the source text file. A simple replacement of all newline
+        # characters with a break line tag should take care of formatting.
+
+        # While wrapping in paragraphs would be possible, it's too much work,
+        # I'd rather display the story 1:1 like it was found in the pre tag.
+        content = unicode(element)
+        content = content.replace('\n', '<br />')
+
+        if self.getConfig('non_breaking_spaces'):
+            content = content.replace(' ', '&nbsp;')
+        return content

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import time
+import json
+
+from .. import BeautifulSoup as bs
+#from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+class FictionPadSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','fpad')
+        self.dateformat = "%Y-%m-%dT%H:%M:%SZ"
+        self.is_adult=False
+        self.username = None
+        self.password = None
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL("https://"+self.getSiteDomain()
+                         +"/author/"+m.group('author')
+                         +"/stories/"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fictionpad.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "https://fictionpad.com/author/Author/stories/1234/Some-Title"
+    
+    def getSiteURLPattern(self):
+        # http://fictionpad.com/author/Serdd/stories/4275
+        return r"http(s)?://(www\.)?fictionpad\.com/author/(?P<author>[^/]+)/stories/(?P<id>\d+)"
+
+# <form method="post" action="/signin">
+#     <input name="authenticity_token" type="hidden" value="u+cfdXh46dRnwVnSlmE2B2BFmHgu760paqgBG6KQeos=" />
+#     <input type="hidden" name="remember" value="1">
+#     <strong class="help-start text-center">or with FictionPad</strong>
+#     <label class="control-label hidden-placeholder">Pseudonym or Email Address</label>
+#     <input name="login" class="input-block-level" type="text" placeholder="Pseudonym or Email Address" maxlength="50" required autofocus>
+#     <label class="control-label hidden-placeholder">Password</label>
+#     <input name="password" class="input-block-level" type="password" placeholder="Password" minlength="6" required>
+#     <button type="submit" class="btn btn-primary btn-block">Sign In</button>
+#     <p class="help-end">
+#         <a href="/passwordreset">Forgot your password?</a>
+#     </p>
+# </form>
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['login'] = self.username
+            params['password'] = self.password
+        else:
+            params['login'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['remember'] = '1'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/signin'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['login']))
+
+        ## need to pull empty login page first to get authenticity_token
+        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
+        params['authenticity_token']=soup.find('input', {'name':'authenticity_token'})['value']
+        
+        data = self._postUrl(loginUrl, params)
+
+        if "Invalid email/pseudonym and password combination." in data:
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['login']))
+            raise exceptions.FailedToLogin(loginUrl,params['login'])
+            
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+            if "This is a mature story.  Please sign in to read it." in data:
+                self.performLogin()
+                data = self._fetchUrl(url)
+            
+            find = "wordyarn.config.page = "
+            data = data[data.index(find)+len(find):]
+            data = data[:data.index("</script>")]
+            data = data[:data.rindex(";")]
+            data = data.replace('tables:','"tables":')
+            tables = json.loads(data)['tables']
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        # looks like only one author per story allowed.
+        author = tables['users'][0]
+        story = tables['stories'][0]
+        story_ver = tables['story_versions'][0]
+        print("story:%s"%story)
+        
+        self.story.setMetadata('authorId',author['id'])
+        self.story.setMetadata('author',author['display_name'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/author/'+author['display_name']+'/stories')
+
+        self.story.setMetadata('title',story_ver['title'])
+        self.setDescription(url,story_ver['description'])
+        
+        if not ('assets/story_versions/covers' in story_ver['profile_image_url@2x']):
+            self.setCoverImage(url,story_ver['profile_image_url@2x'])
+
+        self.story.setMetadata('datePublished',makeDate(story['published_at'], self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(story['published_at'], self.dateformat))
+
+        self.story.setMetadata('followers',story['followers_count'])
+        self.story.setMetadata('comments',story['comments_count'])
+        self.story.setMetadata('views',story['views_count'])
+        self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
+        if 'dislikes' in story:
+            self.story.setMetadata('dislikes',int(story['dislikes']))
+
+        if story_ver['is_complete']:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        self.story.setMetadata('rating', story_ver['maturity_level'])
+        self.story.setMetadata('numWords', unicode(story_ver['word_count']))
+
+        for i in tables['fandoms']:
+            self.story.addToList('category',i['name'])
+        
+        for i in tables['genres']:
+            self.story.addToList('genre',i['name'])
+        
+        for i in tables['characters']:
+            self.story.addToList('characters',i['name'])
+
+        for c in tables['chapters']:
+            chtitle = "Chapter %d"%c['number']
+            if c['title']:
+                chtitle += " - %s"%c['title']
+            self.chapterUrls.append((chtitle,c['body_url']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        if not url:
+            data = u"<em>This chapter has no text.</em>"
+        else:
+            data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(u"<div id='story'>"+data+u"</div>")
+        return self.utf8FromSoup(url,soup)
+    
+def getClass():
+    return FictionPadSiteAdapter
+

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time
@@ -38,11 +39,12 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.fictionpress.com','m.fictionpress.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title"
+        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
 
 def getClass():
     return FictionPressComSiteAdapter

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import time
@@ -42,10 +43,11 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     @staticmethod
     def getSiteDomain():
-        return 'www.ficwad.com'
+        return 'ficwad.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.ficwad.com/story/137169"
+        return "http://ficwad.com/story/1234"
 
     def getSiteURLPattern(self):
         return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"
@@ -61,12 +63,12 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             params['password'] = self.getConfig("password")
 
         loginUrl = 'http://' + self.getSiteDomain() + '/account/login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['username']))
         d = self._postUrl(loginUrl,params)
 
         if "Login attempt failed..." in d:
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['username']))
             raise exceptions.FailedToLogin(url,params['username'])
             return False
@@ -79,11 +81,15 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         # metadata and chapter list
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
+            data = self._fetchUrl(url)
+            # non-existent/removed story urls get thrown to the front page.
+            if "<h2>Welcome to FicWad</h2>" in data:
+                raise exceptions.StoryDoesNotExist(self.url)
+            soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -96,7 +102,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             # normalize story URL on chapter list.
             self.story.setMetadata('storyId',storya['href'].split('/',)[2])
             url = "http://"+self.getSiteDomain()+storya['href']
-            logging.debug("Normalizing to URL: "+url)
+            logger.debug("Normalizing to URL: "+url)
             self._setURL(url)
             try:
                 soup = bs.BeautifulSoup(self._fetchUrl(url))
@@ -114,7 +120,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # title - first h4 tag will be title.
         titleh4 = soup.find('h4')
-        self.story.setMetadata('title', titleh4.a.string)
+        self.story.setMetadata('title', stripHTML(titleh4.a))
 
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/author/\d+"))
@@ -124,7 +130,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # description
         storydiv = soup.find("div",{"id":"story"})
-        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p.string)
+        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p)
         #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
 
         # most of the meta data is here:
@@ -201,7 +207,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
 
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -17,10 +17,10 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
 import json
 
 from .. import BeautifulSoup as bs
@@ -41,6 +41,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
         self.is_adult = False
         
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
     @staticmethod
     def getSiteDomain():
         return 'www.fimfiction.net'
@@ -50,11 +54,12 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway
         return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
 
     def getSiteURLPattern(self):
-        return r"http://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
+        return r"https?://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
         
     def extractChapterUrlsAndMetadata(self):
         
@@ -78,8 +83,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
             apiData = json.loads(apiResponse)
             
-            # Unfortunately, we still need to load the story index page to parse the characters
-            data = self._fetchUrl(self.url)
+            # Unfortunately, we still need to load the story index
+            # page to parse the characters.  And chapters, now, too.
+            data = self.do_fix_blockquotes(self._fetchUrl(self.url))
+            soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -89,13 +96,12 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if "/images/missing_story.png" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
+        # Can cause problems if a missing story is referenced in a comment.
+        # Shouldn't be needed anyway.
+        # if "/images/missing_story.png" in data:
+        #     raise exceptions.StoryDoesNotExist(self.url)
         
-        if "Invalid story id" in apiData.values():
-            raise exceptions.StoryDoesNotExist(self.url)
-        
-        if "This story has been marked as having adult content." in data:
+        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
         if self.password:
@@ -109,59 +115,189 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             else:
                 raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
          
+        if "Invalid story id" in apiData.values():
+            raise exceptions.StoryDoesNotExist(self.url)
+        
         storyMetadata = apiData["story"]    
             
-        self.story.setMetadata("title", storyMetadata["title"])
+        ## Title
+        a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # self.story.setMetadata("title", storyMetadata["title"])
+        # if not storyMetadata["title"]:
+        #     raise exceptions.FailedToDownload("%s doesn't have a title in the API.  This is a known fimfiction.net bug with titles containing ."%self.url)
+        
         self.story.setMetadata("author", storyMetadata["author"]["name"])
         self.story.setMetadata("authorId", storyMetadata["author"]["id"])
         self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
+
+        # chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
         
-        chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
-        for chapter in chapters:
-            self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
-        self.story.setMetadata("numChapters", len(self.chapterUrls))
+        # ## this is bit of a kludge based on the assumption all the
+        # ## 'bad' chapters will be at the end.
+        # ## limit down to the number of chapters reported by chapter_count.
+        # chapters = chapters[:storyMetadata["chapter_count"]] 
         
+        # for chapter in chapters:
+        #     self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
+        # self.story.setMetadata("numChapters", len(self.chapterUrls))
+
+        for chapter in soup.findAll('a',{'class':'chapter_link'}):
+            self.chapterUrls.append((stripHTML(chapter), 'http://'+self.host+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
         # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
-        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
+        # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
         # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
         # status to leave unchanged.
-        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
+        # Nov2012 - 'On Hiatus' is now passed, too.  It's easy now for users to change/remove if they want
+        # with replace_metadata
+        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata("status", status)
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
-            
+
+        ## Warnings aren't included in the API.
+        bottomli = soup.find('li',{'class':'bottom'})
+        if bottomli:
+            bottomspans = bottomli.findAll('span')
+            # the first span in bottom is the rating, obtained above.
+            if bottomspans and len(bottomspans) > 1:
+                for warning in bottomspans[1:]:
+                    self.story.addToList('warnings',warning.string)
+                
+        
         for category in storyMetadata["categories"]:
             if storyMetadata["categories"][category]:
                 self.story.addToList("genre", category) 
 
-        self.story.addToList("category", "My Little Pony")
-        
         self.story.setMetadata("numWords", str(storyMetadata["words"]))
         
-        
         # fimfic is the first site with an explicit cover image.
-        if self.getConfig('include_images') and "image" in storyMetadata.keys():
-            coverurl = storyMetadata["image"]
-            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
+        if "image" in storyMetadata.keys():
+            if "full_image" in storyMetadata:
+                coverurl = storyMetadata["full_image"]
+            else:
+                coverurl = storyMetadata["image"]
+            if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
-            self.story.addImgUrl(self,self.url,coverurl,self._fetchUrlRaw,cover=True)
-            
-        self.setDescription(self.url, storyMetadata["description"])
-        
+
+            self.setCoverImage(self.url,coverurl)
+
+        # fimf has started including extra stuff inside the description div.
+        descdivstr = u"%s"%soup.find("div", {"class":"description"})
+        hrstr=u"<hr />"
+        descdivstr = u'<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        self.setDescription(self.url,descdivstr)
+
+        # Can't trust dates from API anymore I'm told.
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
         
-        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
-        for character in [character_icon["title"] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
-            self.story.addToList("characters", character)
+        oldestChapter = None
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for index, chapterDate in enumerate(soup.findAll('span', {'class':'date'})):
+            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(date,self.dateformat)
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+                self.newestChapterNum = index
+
+        self.story.setMetadata("dateUpdated", newestChapter)
+        
+        pubdatetag = soup.find('span', {'class':'date_approved'})
+        if pubdatetag is None:
+            self.story.setMetadata("datePublished", oldestChapter)            
+        else:
+            pubdateraw = pubdatetag('span')[1].text
+            datestripped=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",pubdateraw.strip())
+            pubDate = makeDate(datestripped,self.dateformat)
+            self.story.setMetadata("datePublished", pubDate)
             
+        chars = soup.find("div", {"class":"inner_data"})
+        # fimfic stopped putting the char name on or around the char
+        # icon now for some reason.  Pull it from the image name with
+        # some heuristics.
+        for character in [character_icon["src"] for character_icon in chars.findAll("img", {"class":"character_icon"})]:
+            # //static.fimfiction.net/images/characters/twilight_sparkle.png
+            # 5th split /, remove last four, replace _, capitolize every word(title())
+            char = character.split('/')[5][:-4].replace('_',' ').title()
+            if char == 'Oc':
+                char = "OC"
+            if char == 'Cmc':
+                char = "Cutie Mark Crusaders"
+            self.story.addToList("characters", char)
             
+        # extra site specific metadata
+        extralist = ["likes","dislikes","views","total_views","short_description"]
+        for metakey in extralist:
+            if metakey in storyMetadata:
+                value = storyMetadata[metakey]
+                if not isinstance(value,basestring):
+                    value = unicode(value)
+                self.story.setMetadata(metakey, value)
+
+        ## Groups and sequels code from FaceDeer
+        allGroupLists = soup.findAll('ul', {'id':'story_group_list'})
+        for groupList in allGroupLists:
+            for groupName in groupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
+                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
+
+        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
+        if not sequelStoryHeader == None:
+            sequelContainer = sequelStoryHeader.parent.parent
+            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
+                self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
+                self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
+                
+        #The link to the prequel is embedded in the description text, so erring
+        #on the side of caution and wrapping this whole thing in a try block.
+        #If anything goes wrong this probably wasn't a valid prequel link.
+        try:
+            description = soup.find('div', {'class':'description'})
+            firstHR = description.find("hr")
+            nextSib = firstHR.nextSibling
+            if "This story is a sequel to" in nextSib.string:
+                link = nextSib.nextSibling
+                if link.name == "a":
+                    self.story.setMetadata("prequelUrl", 'http://'+self.host+link["href"])
+                    self.story.setMetadata("prequel", stripHTML(link))
+        except:
+            pass
+        
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+
+    def do_fix_blockquotes(self,data):
+        if self.getConfig('fix_fimf_blockquotes'):
+            # <p class="double"><blockquote>
+            # </blockquote></p>
+            # include > in re groups so there's always something in the group.
+            data = re.sub(r'<p([^>]*>\s*)<blockquote([^>]*>)',r'<blockquote\2<p\1',data)
+            data = re.sub(r'</blockquote(>\s*)</p>',r'</p\1</blockquote>',data)
+        return data
+        
     def getChapterText(self, url):
-        logging.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
+        logger.debug('Getting chapter text from: %s' % url)
+
+        data = self.do_fix_blockquotes(self._fetchUrl(url))
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -36,11 +37,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -49,7 +45,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
@@ -66,8 +61,9 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'finestories.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
+        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
@@ -91,17 +87,17 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             params['theusername'] = self.getConfig("username")
             params['thepassword'] = self.getConfig("password")
         params['rememberMe'] = '1'
-        params['page'] = 'http://finestories.com/'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['theusername']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "My Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['theusername']))
             raise exceptions.FailedToLogin(url,params['theusername'])
             return False
@@ -114,7 +110,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -140,7 +136,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"/a/\w+"))
@@ -186,6 +182,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
             i = a.parent.text.split('(')[1].split(')')[0]
             self.setSeries(a.text, i)
+            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
         except:
             pass
         try:
@@ -232,7 +229,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
@@ -259,8 +256,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
                 last[len(last)-1]=last[len(last)-1].append(next)
                 div.append(div1)
             
-            
-            
         # removing all the left-over stuff    
         for a in div.findAll('span'):
             a.extract()

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,8 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        self.section=self.parsedUrl.path.split('/',)[1]
              
         # normalized story URL.
         if "malfoymanor" in self.parsedUrl.netloc:
@@ -60,11 +61,6 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','gech')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%b/%Y"
@@ -78,6 +74,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
 
@@ -105,17 +102,17 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         params['cookiecheck'] = '1'
         params['submit'] = 'Submit'
     
-        if "enchant" in self.story.getMetadata('section'):
+        if "enchant" in self.section:
             loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
         else:
             loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -137,7 +134,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -162,7 +159,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -185,7 +182,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -196,7 +193,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -231,6 +228,9 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -267,7 +267,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -279,9 +279,17 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
-            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('div',{'id':'sort'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
         except:
             # I find it hard to care if the series parsing fails
             pass
@@ -289,10 +297,10 @@ class GrangerEnchantedCom(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'id' : 'story1'})
 

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,12 +38,11 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only psid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
@@ -56,8 +56,9 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234 http://harrypotterfanfiction.com/viewstory.php?psid=5678"
+        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("harrypotterfanfiction.com/viewstory.php?psid=")+r"\d+$"
@@ -73,7 +74,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -91,7 +92,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
         if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
@@ -130,59 +131,59 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Finding the metadata is a bit of a pain.  Most of the meta
         ## data is in a center.table without a bgcolor.
-        for center in soup.findAll('center'):
-            table = center.find('table',{'bgcolor':None})
-            if table:
-                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-                # Rating: 12+ Story Reviews: 3
-                # Chapters: 3
-                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-                # Status: Completed
-                # First Published: 2010.09.02
-                # Last Published Chapter: 2010.09.28
-                # Last Updated: 2010.09.28
-                # Favorite Story Of: 1 users
-                # Warnings: Scenes of a Mild Sexual Nature
+        #for center in soup.findAll('center'):
+        table = soup.find('table',{'class':'storymaininfo'})
+        if table:
+            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+            # Rating: 12+ Story Reviews: 3
+            # Chapters: 3
+            # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
+            # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
+            # Status: Completed
+            # First Published: 2010.09.02
+            # Last Published Chapter: 2010.09.28
+            # Last Updated: 2010.09.28
+            # Favorite Story Of: 1 users
+            # Warnings: Scenes of a Mild Sexual Nature
 
-                m = re.match(r".*?Status: Completed.*?",metastr)
-                if m:
-                    self.story.setMetadata('status','Completed')
-                else:
-                    self.story.setMetadata('status','In-Progress')
+            m = re.match(r".*?Status: Completed.*?",metastr)
+            if m:
+                self.story.setMetadata('status','Completed')
+            else:
+                self.story.setMetadata('status','In-Progress')
 
-                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-                if m:
-                    self.story.setMetadata('rating', m.group(1))
+            m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
+            if m:
+                self.story.setMetadata('rating', m.group(1))
 
-                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('genre',g)
-        
-                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('characters',g)
-        
-                m = re.match(r".*?Warnings: (.+).*?",metastr)
-                if m:
-                    for w in m.group(1).split(','):
-                        if w != 'Now Warnings':
-                            self.story.addToList('warnings',w)
-        
-                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
+            m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('genre',g)
+    
+            m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('characters',g)
+    
+            m = re.match(r".*?Warnings: (.+).*?",metastr)
+            if m:
+                for w in m.group(1).split(','):
+                    if w != 'Now Warnings':
+                        self.story.addToList('warnings',w)
+    
+            m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
 
-                # Updated can have more than one space after it. <shrug>
-                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
+            # Updated can have more than one space after it. <shrug>
+            m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         ## most adapters use BeautifulStoneSoup here, but non-Stone
         ## allows nested div tags.

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HennethAnnunNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HennethAnnunNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','htan')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.henneth-annun.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "We're sorry. This story is not available." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: This story is not available.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h2', {'id':'page_heading'})
+        self.story.setMetadata('title',stripHTML(a))
+
+        # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
+        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        self.story.setMetadata('numWords', soup.find('tr', {'class':'foot'}).findAll('td')[1].text)
+
+        self.setDescription(url,soup.find('div', {'id':'summary'}))
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        info = soup.find('div', {'id':'storyinformation'})
+        labels=info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+                
+            if 'Completion' in label:
+                if 'Complete' in value.string:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Era:' in label:
+                self.story.addToList('category',value.string)
+
+            if 'Genre' in label:
+                self.story.addToList('genre',value.string)
+                
+        labels=info.findAll('strong')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if 'Author' in label:
+                value=value.nextSibling
+                self.story.setMetadata('authorId',value['href'].split('=')[1])
+                self.story.setMetadata('authorUrl','http://'+self.host+'/'+value['href'])
+                self.story.setMetadata('author',value.string)
+                
+            if 'Post' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated:' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+        for char in soup.findAll('a', href=re.compile(r"/resources/bios_view.cfm\?scid=\d+")):
+            self.story.addToList('characters',stripHTML(char))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'block chapter'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','hlf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Highlander")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y"
@@ -69,6 +65,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'hlfiction.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -81,7 +78,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -109,7 +106,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
 
         # Find the chapters:
         chapters=soup.find('select')
@@ -210,6 +207,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -220,7 +218,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','hpfdm') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d" # XXX
@@ -74,6 +70,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'www.hpfandom.net' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
 
@@ -84,7 +81,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -185,21 +182,22 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
             value = td.nextSibling.string
             #print("\nlabel:%s\nvalue:%s\n"%(label,value))
 
-            if 'Category' in label:
+            if 'Category' in label and value:
                 cats = td.parent.findAll('a',href=re.compile(r'categories.php'))
                 catstext = [cat.string for cat in cats]
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
-            if 'Characters' in label:
+            if 'Characters' in label and value: # this site can have Character label with no
+                                                # values, apparently.  Others as a precaution.
                 for char in value.split(','):
                     self.story.addToList('characters',char.strip())
 
-            if 'Genre' in label:
+            if 'Genre' in label and value:
                 for genre in value.split(','):
                     self.story.addToList('genre',genre.strip())
 
-            if 'Warnings' in label:
+            if 'Warnings' in label and value:
                 for warning in value.split(','):
                     if warning.strip() != 'none':
                         self.story.addToList('warnings',warning.strip())
@@ -213,7 +211,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         # There's no good wrapper around the chapter text. :-/

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','hpffa')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"
@@ -69,6 +65,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.hpfanficarchive.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
 
@@ -81,7 +78,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -103,7 +100,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -131,23 +128,22 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # <span class="label">Rated:</span> NC-17<br /> etc
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
-            value = labelspan.nextSibling
+            val = labelspan.nextSibling
+            value = unicode('')
+            while val and not defaultGetattr(val,'class') == 'label':
+                value += unicode(val)
+                val = val.nextSibling
             label = labelspan.string
+            #print("label:%s\nvalue:%s"%(label,value))
 
             if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,value)
 
             if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+                self.story.setMetadata('rating', stripHTML(value))
 
             if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+                self.story.setMetadata('numWords', stripHTML(value))
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -164,13 +160,18 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
             if 'Warnings' in label:
                 warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
-                if 'Yes' in value:
+                if 'Yes' in stripHTML(value):
                     self.story.setMetadata('status', 'Completed')
                 else:
                     self.story.setMetadata('status', 'In-Progress')
@@ -197,6 +198,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -207,7 +209,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ike')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","InuYasha")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"
@@ -69,6 +65,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.ik-eternal.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -97,13 +94,13 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -125,7 +122,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -163,7 +160,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -186,7 +183,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -272,7 +269,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -27,11 +28,11 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return SVUFictionComAdapter
+    return ImagineEFicComAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class SVUFictionComAdapter(BaseSiteAdapter):
+class ImagineEFicComAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -47,23 +48,24 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','svuf')
+        self.story.setMetadata('siteabbrev','ime')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
+        self.dateformat = "%Y.%m.%d"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'svufiction.com'
+        return 'imagine.e-fic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -92,13 +94,13 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -107,20 +109,20 @@ class SVUFictionComAdapter(BaseSiteAdapter):
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
-    
+
         if self.is_adult or self.getConfig("is_adult"):
             # Weirdly, different sites use different warning numbers.
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=6"
+            addurl = "&ageconsent=ok&warning=4"
         else:
             addurl=""
 
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -129,12 +131,12 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url)
-                
+
         m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
@@ -145,7 +147,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -167,22 +169,15 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        pt = soup.find('div', {'class' : 'title'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        if a.text != "":
-            self.story.setMetadata('title',a.text)
-        else:
-            self.story.setMetadata('title','Banner')
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
-        rating=pt.text.split('[')[1].split(']')[0]
-        self.story.setMetadata('rating', rating)
-
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
@@ -199,39 +194,61 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 return d[k]
             except:
                 return ""
+                
 
-        bottom=soup.find('div', {'class' : 'bottom'}).text
-        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
-        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
 
-        
-        content=soup.find('div', {'class' : 'content'})
-        value=content.find('h1').nextSibling
-        svalue = ""
-        while not defaultGetattr(value,'class') == 'smaller':
-            svalue += str(value)
-            value = value.nextSibling
-        self.setDescription(url,svalue)
-        
-        status = content.text.split('Completed: ')[1]
-        if 'Yes' in status:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-        
-        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-            
-        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-            
-        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
             
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
@@ -248,6 +265,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -258,7 +276,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_indeathnet.py

@@ -0,0 +1,200 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return InDeathNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class InDeathNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/')
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','idn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'indeath.net'
+
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/"
+
+    def getSiteURLPattern(self):
+        # http://www.indeath.net/blog/archive/169-ransom-in-death/
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"/blog/(archive/)?(?P<id>\d+)\-(?P<name>[a-z0-9\-]*)/?$"     
+        
+            
+    def getDateFromComponents(self, postmonth, postday):
+        ym = re.search("Entries\ in\ (?P<mon>January|February|March|April|May|June|July|August|September|October|November|December)\ (?P<year>\d{4})",postmonth)
+        d = re.search("(?P<day>\d{2})\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)",postday)
+        postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
+        return postdate
+
+    def getAuthorData(self):
+        
+        mainUrl = self.url.replace("/archive","")
+        
+        try:
+            maindata = self._fetchUrl(mainUrl)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        mainsoup = bs.BeautifulSoup(maindata)
+
+        # find first entry
+        e = mainsoup.find('div',{'class':"entry"})
+        
+        # get post author as author
+        d = e.find('div',{'class':"desc"})
+        a = d.find('strong')
+        self.story.setMetadata('author',a.contents[0].string.strip())
+        
+        # Don't seem to be able to get author pages anymore
+        self.story.setMetadata('authorUrl','http://www.indeath.net/')
+        self.story.setMetadata('authorId','0')
+        
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        try:
+            data = self._fetchUrl(url)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+                
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        h = soup.find('a', id="blog_title")
+        t = h.find('span')
+        self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
+        
+        s = t.find('div')        
+        if s != None:
+            self.setDescription(url,s)
+
+        # Get Author from main blog page since it's not reliably on the archive page
+        self.getAuthorData()
+
+        # Find the chapters:
+        chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
+
+        #reverse the list since newest at the top
+        chapters.reverse()
+        
+        # Get date published & updated from first & last entries
+        posttable=soup.find('div', id="main_column")
+        
+        postmonths=posttable.findAll('th', text=re.compile(r'Entries\ in\ '))
+        postmonths.reverse()
+        
+        postdates=posttable.findAll('span', _class="desc", text=re.compile('\d{2}\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'))
+        postdates.reverse()
+        
+        self.story.setMetadata('datePublished',self.getDateFromComponents(postmonths[0],postdates[0]))
+        self.story.setMetadata('dateUpdated',self.getDateFromComponents(postmonths[len(postmonths)-1],postdates[len(postdates)-1]))
+        
+        # Process List of Chapters              
+        self.story.setMetadata('numChapters',len(chapters))
+        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        for x in range(0,len(chapters)):
+            # just in case there's tags, like <i> in chapter titles.
+            chapter=chapters[x]
+            if len(chapters)==1:
+                self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
+            else:
+                ct = stripHTML(chapter)
+                tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+                if tnew:
+                    chaptertitle = tnew.group('newtitle')
+                else:
+                    chaptertitle = ct
+                self.chapterUrls.append((chaptertitle,chapter['href']))
+
+        
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        
+        #chapter=bs.BeautifulSoup('<div class="story"></div>')
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))
+
+        chapter = soup.find("div", "entry_content")
+        
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chapter)
+

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ksa') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Star Trek") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b/%d/%Y" # XXX
@@ -78,6 +74,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'ksarchive.com' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -104,7 +101,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -137,7 +134,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -167,7 +164,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -190,7 +187,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
-            label = labelspan.string
+            label = stripHTML(labelspan)
 
             if 'Summary' in label:
                 ## Everything until the next span class='label'
@@ -215,50 +212,50 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
+                catstext = [stripHTML(cat) for cat in cats]
                 for cat in catstext:
                     # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
                     # tag in the desc once.
                     if cat and cat.strip() in ('Poetry','Essays'):
-                        self.story.addToList('category',cat.string)
+                        self.story.addToList('category',stripHTML(cat))
 
             if 'Characters' in label:
                 self.story.addToList('characters','Kirk')
                 self.story.addToList('characters','Spock')
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
+                charstext = [stripHTML(char) for char in chars]
                 for char in charstext:
-                    self.story.addToList('characters',char.string)
+                    self.story.addToList('characters',stripHTML(char))
 
             ## Not all sites use Genre, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
+                genrestext = [stripHTML(genre) for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
+                    self.story.addToList('genre',stripHTML(genre))
 
             ## In addition to Genre (which is very site specific) KSA
             ## has 'Story Type', which is much more what most sites
             ## call genre.
             if 'Story Type' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
-                genrestext = [genre.string for genre in genres]
+                genrestext = [stripHTML(genre) for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
+                    self.story.addToList('genre',stripHTML(genre))
 
             ## Not all sites use Warnings, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Warnings' in label:
                 warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
+                warningstext = [stripHTML(warning) for warning in warnings]
                 self.warning = ', '.join(warningstext)
                 for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
+                    self.story.addToList('warnings',stripHTML(warning))
 
             if 'Completed' in label:
                 if 'Yes' in value:
@@ -277,7 +274,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
+            series_name = stripHTML(a)
             series_url = 'http://'+self.host+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
@@ -289,6 +286,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -299,7 +297,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -56,11 +57,6 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','lom')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Lord of the Rings")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"
@@ -70,6 +66,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.libraryofmoria.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"
 
@@ -89,7 +86,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -109,7 +106,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -132,12 +129,12 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/a/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -229,6 +226,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -239,7 +237,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_literotica.py

@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+class LiteroticaSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.story.setMetadata('siteabbrev','litero')
+
+        # normalize to first chapter.  Not sure if they ever have more than 2 digits.
+        storyid = self.parsedUrl.path.split('/',)[2]
+        if re.match(r'-ch\d\d$',storyid):
+            storyid = storyid[:-2]+'01'
+        self.story.setMetadata('storyId',storyid)
+        
+        self.origurl = url
+        if "//www.i." in self.origurl:
+            ## accept m(mobile)url, but use www.
+            self.origurl = self.origurl.replace("//www.i.","//www.")
+
+        # normalized story URL.
+        self._setURL(url[:url.index('//')+2]+self.getSiteDomain()\
+                         +"/s/"+self.story.getMetadata('storyId'))
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%m/%d/%y'
+        
+    @staticmethod
+    def getSiteDomain():
+        return 'www.literotica.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.literotica.com', 'www.i.literotica.com']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
+        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title"
+
+    def getSiteURLPattern(self):
+        return r"https?://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        url1 = self.origurl
+        logger.debug("first page URL: "+url1)
+
+        try:
+            data1 = self._fetchUrl(url1)
+            soup1 = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url1)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # author
+        a = soup1.find("span", "b-story-user-y")
+        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'][0])
+        authorurl = a.a['href']
+        if authorurl.startswith('//'):
+            authorurl = self.parsedUrl.scheme+':'+authorurl
+        self.story.setMetadata('authorUrl', authorurl)
+        self.story.setMetadata('author', a.text)
+
+        # get the author page
+        try:
+            dataAuth = self._fetchUrl(authorurl)
+            soupAuth = bs.BeautifulSoup(dataAuth)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(authorurl)
+            else:
+                raise e
+
+        ## site has started using //domain.name/asdf urls remove https?: from front
+        storyLink = soupAuth.find('a', href=url1[url1.index(':')+1:])
+
+        if storyLink is not None:
+            # pull the published date from the author page
+            # default values from single link.  Updated below if multiple chapter.
+            date = storyLink.parent.parent.findAll('td')[-1].text
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
+
+        # find num of pages
+        # find a "3 Pages:" string on the page and parse it
+        pgs = soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0]
+
+        # If there are multiple pages, find and request the last page
+        if "1" != pgs:
+            logger.debug("last page number: "+pgs)
+            try:
+                data2 = self._fetchUrl(url1, {'page': pgs})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    # TODO: Probably should reformat this
+                    raise exceptions.StoryDoesNotExist(url1, {'page': pgs})
+                else:
+                    raise e
+        else:
+            #If we're already on the last page, copy the soup
+            soup2 = soup1
+
+        # parse out the list of chapters
+        chaps = soup2.find('div', id='b-series')
+        if chaps:  # may be one post only
+            #self.chapterUrls = [(ch.a.text, ch.a['href']) for ch in chaps.findAll('li')]
+
+            # if there are chapters, lets pull them and title from the
+            # author page because *this* chapter is omitted from the
+            # list on the last page.
+            row = storyLink.parent.parent.previousSibling
+            while row['class'] != 'ser-ttl':
+                row = row.previousSibling
+
+            seriesTitle = stripHTML(row)
+            if seriesTitle:
+                # this regex is deliberately greedy. We want to get the biggest match before a ':'
+                self.story.setMetadata('title', re.match('(.*):[^:]*$', seriesTitle).group(1))
+            else:
+                self.story.setMetadata('title', soup1.h1.string)
+
+            # now chapter list.  Assumed oldest to newest.
+            self.chapterUrls = []
+            row = row.nextSibling
+                
+            self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+            while row['class'] == 'sl':
+                # pages include full URLs.
+                chapurl = row.a['href']
+                if chapurl.startswith('//'):
+                    chapurl = self.parsedUrl.scheme+':'+chapurl
+                self.chapterUrls.append((row.a.string,chapurl))
+                if not row.nextSibling:
+                    break
+                row = row.nextSibling
+
+            row = row.previousSibling
+            self.story.setMetadata('dateUpdated',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+                
+        else:  # if one post only
+            self.chapterUrls = [(soup1.h1.string, url1)]
+            self.story.setMetadata('title', soup1.h1.string)
+
+        # normalize on first chapter URL.
+        self._setURL(self.chapterUrls[0][1])
+
+        # reset storyId to first chapter.
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+
+        self.story.setMetadata('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)
+        # deliberately not self.setDescription() because it's never HTML.
+        self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
+        
+        # li tags inside div class b-s-story-tag-list
+        for li in soup1.find('div', {'class':'b-s-story-tag-list'}).findAll('a'):
+            self.story.addToList('eroticatags',stripHTML(li))
+
+        return
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5)
+        data1 = self._fetchUrl(url)
+        soup1 = bs.BeautifulSoup(data1)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        story1 = soup1.find('div', 'b-story-body-x').p
+        story1.name='div'
+        story1.append('<br />')
+        storytext = self.utf8FromSoup(url,story1)
+
+        # find num pages
+        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
+        logger.debug("pages: "+str(pgs))
+
+        # get all the pages
+        for i in xrange(2, pgs+1):
+            try:
+                logger.debug("fetching page "+str(i))
+                time.sleep(0.5)
+                data2 = self._fetchUrl(url, {'page': i})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+                story2 = soup2.find('div', 'b-story-body-x').p
+                story2.name='div'
+                story2.append('<br />')
+                storytext += self.utf8FromSoup(url,story2)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        return storytext
+
+
+def getClass():
+    return LiteroticaSiteAdapter
+
+
+

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','lsph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -70,6 +66,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'lumos.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -91,7 +88,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -119,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -216,6 +213,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -226,7 +224,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -42,7 +43,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
         else:
@@ -54,6 +55,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.mediaminer.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+self.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
 
@@ -66,7 +68,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -105,10 +107,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
         if title.has_key('colspan'):
-            titlet = title.text
+            titlet = stripHTML(title)
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
         ## The story title is difficult to reliably parse from the
         ## story pages.  Getting it from the author page is, but costs
@@ -188,7 +190,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data=self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,
@@ -214,7 +216,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
             return self.utf8FromSoup(url,anchor)
         
         else:
-            logging.debug('Using kludgey text find for older mediaminer story.')
+            logger.debug('Using kludgey text find for older mediaminer story.')
             ## Some older mediaminer stories are unparsable with BeautifulSoup.
             ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
             ## Story stuff falls between:

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -27,22 +28,15 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return YourFanfictionComAdapter
+    return MerlinFicDtwinsCoUk
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class YourFanfictionComAdapter(BaseSiteAdapter):
+class MerlinFicDtwinsCoUk(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # yourfanfiction.com blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted as us and offered to 'whitelist our IP'.  Clearly,
-        # that wouldn't work, but it does let me do this in good
-        # conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
-
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -54,29 +48,65 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','yff')
+        self.story.setMetadata('siteabbrev','mrfd')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
+        self.dateformat = "%b %d, %Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.yourfanfiction.com'
+        return 'merlinfic.dtwins.co.uk'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -92,7 +122,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -102,20 +132,12 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((&ageconsent=ok)?&warning=\d+)'",data)
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -123,10 +145,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 # link and reload data.
                 addurl = m.group(1)
                 # correct stupid & error in url.
-                # explicitly put ageconsent because google appengine regexp doesn't include it for some reason.
-                addurl = addurl.replace("&","&")+'&ageconsent=ok'
+                addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -141,23 +162,15 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
-        # because for some reason, this works while simple 'print data' errors on ascii conversion.
-        # loopdata = data
-        # chklen=5000
-        # while len(loopdata) > 0:
-        #     if len(loopdata) < 5000:
-        #         chklen = len(loopdata)
-        #     logging.info("loopdata: %s" % loopdata[:chklen])
-        #     loopdata = loopdata[chklen:]
-            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
+        # print data
 
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -191,11 +204,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while value and not defaultGetattr(value,'class') == 'label':
+                while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                # sometimes poorly formated desc (<p> w/o </p>) leads
-                # to all labels being included.
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -214,19 +225,19 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
                 for char in chars:
                     self.story.addToList('characters',char.string)
+                    
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-                    
-            if 'Tags' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=7'))
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -240,8 +251,6 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
         try:
@@ -260,6 +269,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -270,7 +280,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','mw') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Queer as Folk") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y" # XXX
@@ -74,6 +70,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'www.midnightwhispers.ca' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -100,7 +97,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -133,7 +130,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -264,6 +261,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -274,7 +272,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
         soup = bs.BeautifulStoneSoup(data,

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -59,25 +60,25 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','mgln') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y" # XXX
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction.mugglenet.com' # XXX
+        # The site domain.
+        return 'fanfiction.mugglenet.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']
+
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -101,13 +102,13 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId')
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -130,7 +131,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -169,7 +170,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -192,7 +193,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -310,6 +311,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -320,7 +322,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ntlb')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","West Wing")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m-%d-%y"
@@ -72,8 +68,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.national-library.net','national-library.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+        # ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: 
+        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
@@ -85,7 +83,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -106,7 +104,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
@@ -154,6 +152,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
                 for char in value.string.split(', '):
                     self.story.addToList('characters',char)
                     
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
             if 'Warnings' in label:
                 for warning in value.string.split(', '):
                     self.story.addToList('warnings',warning)
@@ -163,6 +165,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
             
             if 'Series' in label:
                 self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
@@ -178,7 +181,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class NCISFicComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ncisf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m-%d-%y"
@@ -72,6 +68,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.ncisfic.com','ncisfic.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
 
@@ -85,7 +82,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -106,7 +103,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
@@ -154,6 +151,10 @@ class NCISFicComAdapter(BaseSiteAdapter):
                 for char in value.string.split(', '):
                     self.story.addToList('characters',char)
                     
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
             if 'Warnings' in label:
                 for warning in value.string.split(', '):
                     self.story.addToList('warnings',warning)
@@ -164,6 +165,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
             if 'Series' in label:
                 if "No Series" not in value.nextSibling.string:
                     self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                    self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
@@ -185,7 +187,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -27,11 +28,11 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return NCISFictionComAdapter
+    return NCISFictionNetAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class NCISFictionComAdapter(BaseSiteAdapter):
+class NCISFictionNetAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -47,7 +48,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL("http://"+self.getSiteDomain()\
@@ -56,11 +57,6 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ncisfn')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%Y"
@@ -68,13 +64,22 @@ class NCISFictionComAdapter(BaseSiteAdapter):
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.ncisfiction.com'
+        return 'www.ncisfiction.net'
 
+    ## Changed from www.ncisfiction.com to www.ncisfiction.net Oct
+    ## 2012 due to the ncisfiction.com domain expiring.  Still accept
+    ## .com domains for existing updates, etc.
+    
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ncisfiction.net','www.ncisfiction.com']
+
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
 
     def getSiteURLPattern(self):
-        return "http://"+self.getSiteDomain()+r'/(chapters|story)?.php\?stid=\d+'
+        return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+'
        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
@@ -83,7 +88,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -157,6 +162,11 @@ class NCISFictionComAdapter(BaseSiteAdapter):
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
+            if 'Pairing' in label:
+                ships = value.findAll('a')
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
             if 'Genre' in label:
                 genres = value.findAll('a')
                 for genre in genres:
@@ -179,7 +189,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
             series_name = a.find('a').string
             i = a.text.split('#')[1]
             self.setSeries(series_name, i)
-            
+            self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href'])
         except:
             # I find it hard to care if the series parsing fails
             pass
@@ -187,7 +197,7 @@ class NCISFictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_netraptororg.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -27,11 +28,9 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return PrisonBreakFicNetAdapter
+    return NetRaptorOrgAdapter
 
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PrisonBreakFicNetAdapter(BaseSiteAdapter):
+class NetRaptorOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -47,41 +46,35 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pbf')
-
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Prison Break")
+        self.story.setMetadata('siteabbrev','netrap')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d/%m/%y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.prisonbreakfic.net'
+        return 'netraptor.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -90,6 +83,10 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+
+
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -98,19 +95,20 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -123,10 +121,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 return d[k]
             except:
                 return ""
-                
-# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
+				
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
@@ -149,16 +144,18 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
+                charstext = [char.string for char in chars]
+                for char in charstext:
                     self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
@@ -183,7 +180,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -192,6 +189,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -202,10 +200,9 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -51,7 +52,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -60,11 +61,6 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','nfa') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y" # XXX
@@ -78,6 +74,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'nfacommunity.com' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -104,7 +101,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -137,7 +134,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -159,7 +156,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -268,6 +265,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -278,7 +276,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','nha')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Buffy the Vampire Slayer")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = " %m/%d/%y"
@@ -70,6 +66,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'nha.magical-worlds.us'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -79,7 +76,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -116,8 +113,9 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 		
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                self.story.setMetadata('title',a.string)
+            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
+                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 
@@ -146,14 +144,14 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         for cat in cats:
             self.story.addToList('category',cat.string)
 					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        a = info.find('a', href=re.compile(r'viewuser.php'))
         val = a.nextSibling
         svalue = ""
         while not defaultGetattr(val) == 'br':
             val = val.nextSibling
         val = val.nextSibling
         while not defaultGetattr(val) == 'br':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
         
@@ -202,7 +200,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
 

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NickAndGregNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NickAndGregNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nag')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.nickandgreg.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&i=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        for chapter in chapters.findAll('option'):
+            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
+            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+        
+        self.setDescription(url,div.find('br').nextSibling)
+        
+        a=div.text.split('Rating:')
+        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
+        
+        a=div.text.split('Characters:')
+        if len(a) == 2: 
+        	for char in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('characters',char)
+        	    
+        a=div.text.split('Genres:')
+        if len(a) == 2: 
+        	for genre in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('genre',genre)
+        	    
+        a=div.text.split('Warnings:')
+        if len(a) == 2: 
+        	for warn in a[1].split(' -')[0].split(', '):
+        	    if 'none' not in warn:
+        	        self.story.addToList('warnings',warn)
+        	    
+        a=div.text.split('Completed:')
+        if len(a) ==2:
+            if 'Yes' in a[1]:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+                
+        a=div.text.split('Published:')
+        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+        
+        a=div.text.split('Updated:')
+        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        # wrap a div around it.
+        divsoup = bs.BeautifulStoneSoup('<div class="story"></div>',
+                                        selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        div = divsoup.find('div')
+        div.append(soup.find('table', {'class' : 'tblborder6'}))
+
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nocturnallightnet.py

@@ -0,0 +1,177 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return NocturnalLightNetAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class NocturnalLightNetAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'nln'
+    SITE_DOMAIN = 'nocturnal-light.net'
+    BASE_URL = 'http://' + SITE_DOMAIN + '/fanfiction/'
+    STORY_URL_TEMPLATE = BASE_URL + 'story/%s'
+    AUTHORS_URL_TEMPLATE = BASE_URL + 'authors/%s'
+
+    DATETIME_FORMAT = '%m-%d-%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        url_tokens = self.parsedUrl.path.split('/')
+        story_id = url_tokens[url_tokens.index('story') + 1]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return NocturnalLightNetAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.STORY_URL_TEMPLATE[:-2]) + r'\d+.*$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        # "storycontent" is found in a single-chapter story
+        author_anchor = soup.find('div', id=lambda id: id in ('main', 'storycontent')).h1.a
+        self.story.setMetadata('author', author_anchor.string)
+
+        url_tokens = author_anchor['href'].split('/')
+        author_id = url_tokens[url_tokens.index('authors')+1]
+        self.story.setMetadata('authorId', author_id)
+        self.story.setMetadata('authorUrl', self.AUTHORS_URL_TEMPLATE % author_id)
+
+        chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/'))
+        for chapter_anchor in chapter_anchors:
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((chapter_anchor.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+        for listbox in soup('div', {'class': 'listbox'}):
+            url_tokens = listbox.a['href'].split('/')
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if story_id == url_tokens[url_tokens.index('story')+1]:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = listbox.a.string
+        self.story.setMetadata('title', title)
+
+        # No chapter anchors found in the original story URL, so the story has
+        # only a single chapter.
+        if not chapter_anchors:
+            self.chapterUrls.append((title, self.url))
+
+        for b_tag in listbox('b'):
+            key = b_tag.string.strip(':')
+            try:
+                value = b_tag.nextSibling.string.replace('•', '').strip(': ')
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(b_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(author_url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Pop last break line tag
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'b']):
+                    if sibling.name == 'b':
+                        break
+
+                    self.story.addToList('category', sibling.string)
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+                # Also parse reviews number which lies right after the chapters
+                # section
+                reviews_anchor = b_tag.findNextSibling('a')
+                reviews = reviews_anchor.string.split(' ')[1].strip('()')
+                self.story.setMetadata('reviews', reviews)
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Date Added':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Last Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value.split()[0])
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='storytext'))

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -56,11 +57,6 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','osph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -70,6 +66,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'occlumency.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -99,13 +96,13 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Logout" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -117,7 +114,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -161,7 +158,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'class' : 'border'}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.text)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 
@@ -250,7 +247,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 		
         data = self._fetchUrl(url)
         data = data.replace('<div align="left"', '<div align="left">')

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','odf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","One Direction")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y"
@@ -74,6 +70,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']
 
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -102,13 +99,13 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
         d = self._fetchUrl(loginUrl, params)
     
         if "Member Account" not in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -130,7 +127,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -161,7 +158,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -195,7 +192,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
@@ -248,6 +245,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -258,7 +256,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2, urllib, cookielib
 
@@ -47,7 +48,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
@@ -55,11 +56,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','phs')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d %Y"
@@ -69,6 +65,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.phoenixsong.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"
 
@@ -95,12 +92,12 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         params['login'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/users/processlogin.php'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['txtusername']))
         d = self._fetchUrl(loginUrl, params)
     
         if 'Please login to continue.' in d : #Member Account
-            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['txtusername']))
             raise exceptions.FailedToLogin(url,params['txtusername'])
             return False
@@ -113,9 +110,11 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
+            if self.getConfig('force_login'):
+                self.performLogin(url)
             data = self._fetchUrl(url)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -127,7 +126,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url)
-			
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -138,7 +136,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         ## Title
         b = soup.find('div', {'id' : 'nav25'})
         a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
         a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))
@@ -213,7 +211,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
 	# grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -0,0 +1,300 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PommeDeSangComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PommeDeSangComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # pommedesang.com has two 'sections', shown in URL as
+        # 'efiction' and 'sds' that change how things should be
+        # handled.
+        # http://pommedesang.com/efiction/viewstory.php?sid=1234
+        # http://pommedesang.com/sds/viewstory.php?sid=1234
+        self.section=self.parsedUrl.path.split('/',)[1]
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pmds')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'efiction' in self.section:
+            self.dateformat = "%b %d, %Y"
+        else:
+            self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'pommedesang.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/sds/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(efiction|sds)?/viewstory.php\?sid=\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/'+self.section+'/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile('viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,8 +48,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        logging.info(self.parsedUrl.netloc)
+        
         # normalized story URL.
         if "explicit" in self.parsedUrl.netloc:
             self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -60,11 +60,6 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','pffa')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","My Little Pony")
-            
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
@@ -74,8 +69,9 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -96,7 +92,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -117,7 +113,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -140,7 +136,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -229,6 +225,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -239,10 +236,9 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
@@ -54,7 +55,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
@@ -62,11 +63,6 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','prtky') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y" # XXX
@@ -76,6 +72,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'fanfiction.portkey.org' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story/1234"
 
@@ -86,7 +83,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         # portkey screws around with using a different URL to set the
         # cookie and it's a pain.  So... cheat!
@@ -252,10 +249,12 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        data = data.replace("HTML>","div>")
+        
+        soup = bs.BeautifulSoup(data)
 
         #print("soup:%s"%soup)
         tag = soup.find('td', {'class' : 'story'})

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib
 import urllib2
@@ -37,11 +38,10 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,8 +55,9 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.potionsandsnitches.net','potionsandsnitches.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234 http://potionsandsnitches.net/fanfiction/viewstory.php?sid=5678"
+        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"
@@ -64,7 +65,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -82,7 +83,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -142,7 +143,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
                 charstext = [char.string for char in chars]
                 for char in charstext:
-                    if char == "!Snape and Harry (required)":
+                    if "Snape and Harry (required)" in char:
                         self.story.addToList('characters',"Snape")
                         self.story.addToList('characters',"Harry")
                     else:
@@ -162,12 +163,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%b %d %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.
@@ -182,6 +182,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -192,7 +193,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_potterficscom.py

@@ -0,0 +1,243 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return PotterFicsComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterFicsComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL. gets rid of chapter if there, left with chapter index URL
+            nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','potficscom')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.potterfics.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://www.potterfics.com/historias/12345 http://www.potterfics.com/historias/12345/capitulo-1 "
+
+    def getSiteURLPattern(self):
+        #http://www.potterfics.com/historias/127583
+        #http://www.potterfics.com/historias/127583/capitulo-1
+        #http://www.potterfics.com/historias/127583/capitulo-4
+        #http://www.potterfics.com/historias/92810              -> Complete story
+        #http://www.potterfics.com/historias/111194             -> Complete, single chap
+        p = re.escape("http://"+self.getSiteDomain()+"/historias/")+\
+            r"(?P<id>\d+)(/capitulo-(?P<ch>\d+))?/?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        #this converts '/historias/12345' to 'http://www.potterfics.com/historias/12345'
+        def makeAbsoluteURL(url):
+            if url[0] == '/':
+                url = 'http://'+self.getSiteDomain()+url
+            return url
+
+        #use this to get month numbers from Spanish months
+        SpanishMonths = {
+            'enero'      : '01',
+            'febrero'    : '02',
+            'marzo'      : '03',
+            'abril'      : '04',
+            'mayo'       : '05',
+            'junio'      : '06',
+            'julio'      : '07',
+            'agosto'     : '08',
+            'septiembre' : '09',
+            'octubre'    : '10',
+            'noviembre'  : '11',
+            'diciembre'  : '12'
+            }
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        #deal with adult content warnings - doesn't seem to apply to this site
+
+        #set constant meta for this site:
+        #Set Language = Spanish
+        self.story.setMetadata('language', 'Spanish')
+        #Set Category = Harry Potter
+        #  This is better done in plugin-defaults.ini and defaults.ini
+        #  by adding a section for this site with the line:
+        #  extracategories:Harry Potter
+        #self.story.addToList('category','Harry Potter')
+        
+        #get the rest of the meta
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        #self closing br and img present!
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','img'))
+    
+        #we want the second table directly under the body, contains all the metadata
+        table = soup.html.body.findAll('table', recursive=False)[1]
+        #within that, we want the second row, first cell
+        cell = table.tr.findNextSibling('tr').td
+
+        #find first metadata block
+        mb = cell.div.findNextSibling('div')
+        #Get meta...
+        self.story.setMetadata('title', stripHTML(mb.b))
+        #strip out brackets on rating
+        self.story.setMetadata('rating', mb.span.string[1:-1])
+        #Completion status is denoted by the presence of this image:
+        if mb.find('img',title="Historia terminada"):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        #find next metadata block
+        #author details
+        mb = mb.findNextSibling('div')
+        self.story.setMetadata('author', mb.b.a.string.strip())
+        self.story.setMetadata('authorUrl', makeAbsoluteURL(mb.b.a['href']))
+        self.story.setMetadata('authorId', self.story.getMetadata('authorUrl').split('/')[4])
+        #dates and times
+        mb = mb.find('span')
+        #posted/published = Escrita
+        date = mb.find(text=re.compile('Escrita el ')).strip().split()
+        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
+        month = int(SpanishMonths[date[5].lower()])
+        day = int(date[3])
+        time = date[8].split(':')
+        hour = int(time[0])
+        minute = int(time[1])
+        self.story.setMetadata('datePublished', datetime.datetime(year, month, day, hour, minute))
+        #updated = Actualizada
+        date = mb.find(text=re.compile('Actualizada el ')).strip().split()
+        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
+        month = int(SpanishMonths[date[5].lower()])
+        day = int(date[3])
+        time = date[8].split(':')
+        hour = int(time[0])
+        minute = int(time[1])
+        self.story.setMetadata('dateUpdated', datetime.datetime(year, month, day, hour, minute))
+        
+        mb = mb.span.findNextSibling('span').findNextSibling('span')
+        wc = mb.find(text=re.compile(' palabras en total')).strip()
+        self.story.setMetadata('numWords', wc.split()[0])
+        
+        #then we come to categories and genres. Oh dear. On this site, categories hold everything from genre, to ships, to crossovers.
+        #To make things worse, there is also another genre field, which often holds similar/duplicate info. Links to genre pages do not work
+        #though, so perhaps those will be phased out?
+        #for now, put them all into the genre list
+        links = mb.findAll('a',href=re.compile('/(categorias|generos)/\d+'))
+        genlist = [i.string.strip() for i in links]
+        self.story.extendList('genre',genlist)
+        
+        #get the chapter urls
+        #we can go back to the table cell we found before
+        #get its last element and work backwards to find the last ordered list on the page
+        list = cell.contents[len(cell)-1].findPrevious('ol')
+        chapters = []
+        revs = 0
+        chnum = 0
+        for li in list:
+            chnum += 1
+            chTitle = str(chnum) + '. ' + li.a.b.string.strip()
+            chURL = makeAbsoluteURL(li.a['href'])
+            chapters.append((chTitle,chURL))
+            #Get reviews, add to total
+            revs += int(li.div.a.string.split()[0])
+        
+        self.chapterUrls.extend(chapters)
+        self.story.setMetadata('numChapters', len(chapters))
+        self.story.setMetadata('reviews', revs)
+        
+        #Now for the description... this may be tricky...
+        #if it is there (doesn't have to be), it will be before the chapter list,
+        #separated by a horizontal rule, and after the google ad bar
+        
+        #get list's parent div
+        mb = list.parent
+        #get the div before that, will either be the description, or the google ad bar
+        mb = mb.findPreviousSibling('div')
+        if 'google_ad_client' in str(mb):
+            #couldn't find description, leaving it blank
+            pass
+        else:
+            self.setDescription(url,mb)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr','img'))
+        
+        div = soup.find('div', {'id' : 'cuerpoHistoria'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_potterheadsanonymouscom.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PotterHeadsAnonymousComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pha')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfic.potterheadsanonymous.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ptdc')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","The Pretender")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y"
@@ -73,6 +69,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.pretendercentre.com','pretendercentre.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
 
@@ -94,7 +91,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -114,7 +111,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -137,7 +134,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -232,6 +229,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -242,7 +240,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_psychficcom.py

@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PsychFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PsychFicComAdapter(BaseSiteAdapter): 
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','psyf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.psychfic.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_qafficcom.py

@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return QafFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class QafFicComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','atp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.qaf-fic.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=")+r"\d+$"
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=NC-17"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\s+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/atp/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. 
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification' and value != None:
+                    if "Featured Stories" not in value:
+                        svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            if list.find('a', href=re.compile(r"series.php")) != None:
+                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
+                    # Find Series name from series URL.
+                    series_url = 'http://'+self.host+'/atp/'+series['href']
+                    # use BeautifulSoup HTML parser to make everything easier to find.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+                    i=1
+                    for a in storyas:
+                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                            name=seriessoup.find('div', {'id' : 'pagetitle'})
+                            name.find('a').extract()
+                            self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
+                            i=0
+                            break
+                        i+=1
+                    if i == 0:
+                        break
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -0,0 +1,265 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+from datetime import datetime
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return RestrictedSectionOrgSiteAdapter
+
+class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
+    
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+            
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        
+        # normalized story URL.
+        # get story/file and storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL('http://' + self.getSiteDomain() + '/' + m.group('filestory') + '.php?' + m.group('filestory') + '=' + self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        self.story.setMetadata('siteabbrev','ressec')
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y" # 20 Nov 2005
+        
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'www.restrictedsection.org'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?story=1234 http://"+self.getSiteDomain()+"/file.php?file=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/(?P<filestory>file|story).php\?(file|story)=(?P<id>\d+)$"
+        
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        # one-shot stories use file url instead of story.  'Luckily',
+        # we don't have to worry about one-shots becoming
+        # multi-chapter because ressec is frozen.  Still need 'story'
+        # url for metadata, however.
+        try:
+            if 'file' in url:
+                data = self._postUrlUP(url)
+                soup = bs.BeautifulSoup(data)
+                storya = soup.find('a',href=re.compile(r"^story.php\?story=\d+"))
+                url = 'http://'+self.host+'/'+storya['href'].split('&')[0]  # strip rs_session
+
+                fileas = soup.find('a',href=re.compile(r"^file.php\?file=\d+"))
+                if fileas:
+                    for filea in fileas:
+                        if 'Previous Chapter' in filea.string or 'Next Chapter' in filea.string:
+                            raise exceptions.FailedToDownload(self.getSiteDomain() +" Cannot use chapter url with multi-chapter stories on this site.")
+                    
+            logger.debug("metadata URL: "+url)
+            data = self._fetchUrl(url)
+            # print data
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+        if "Story not found" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Story not found.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # check user/pass on a chapter for multi-chapter
+        if 'file' not in self.url:
+            self._postUrlUP('http://'+self.host+'/'+soup.find('a', href=re.compile(r"^file.php\?file=\d+"))['href'])
+
+        ## Title
+        h2 = soup.find('h2')
+
+        # Find authorid and URL from... author url.
+        a = h2.find('a')
+        ahref = a['href'].split('&')[0]  # strip rs_session
+        
+        self.story.setMetadata('authorId',ahref.split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref)
+        self.story.setMetadata('author',stripHTML(a))
+
+        # title, remove byauthorname.
+        auth=stripHTML(a)
+        title=stripHTML(h2)
+        self.story.setMetadata('title',title[:title.index(" by "+auth)])
+
+        dates = soup.findAll('span', {'class':'date'})
+        if dates: # only for multi-chapter
+            self.story.setMetadata('datePublished', makeDate(stripHTML(dates[0]), self.dateformat))
+            self.story.setMetadata('dateUpdated', makeDate(stripHTML(dates[-1]), self.dateformat))
+
+        words = soup.findAll('span', {'class':'size'})
+        wordcount=0
+        for w in words:
+            wordcount = wordcount + int(w.string[:-6].replace(',',''))
+
+        self.story.setMetadata('numWords',"%s"%wordcount)
+
+        self.story.setMetadata('rating', soup.find('a',href=re.compile(r"^rating.php\?rating=\d+")).string)
+        
+        # other tags
+
+        labels = soup.find('table', {'class':'info'}).findAll('th')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if label != None:
+                
+                if 'Categories' in label:
+                    for g in stripHTML(value).split('\n'):
+                        self.story.addToList('genre',g)
+                
+                if 'Pairings' in label:
+                    for g in stripHTML(value).split('\n'):
+                        self.story.addToList('ships',g)
+                
+                if 'Summary' in label:
+                    self.setDescription(url,stripHTML(value).replace("\n"," ").replace("\r",""))
+                    value.extract() # remove summary incase it contains file URLs.
+
+                if 'Updated' in label: # one-shots only.
+                    print "value:%s"%value
+                    value.find('sup').extract() # remove 'st', 'nd', 'th' ordinals
+                    print "value:%s"%value
+                    date = makeDate(stripHTML(value), '%d %B %Y') # full month name
+                    self.story.setMetadata('datePublished', date)
+
+                if 'Length' in label: # one-shots only.
+                    self.story.setMetadata('numWords',value.string[:-6])
+                    
+        # one-shot.
+        if 'file' in self.url:
+            self.chapterUrls.append((self.story.getMetadata('title'),self.url))
+        else: # multi-chapter
+            # Find the chapters: 'library_storyview.php?chapterid=3
+            chapters=soup.findAll('a', href=re.compile(r"^file.php\?file=\d+"))
+            if len(chapters)==0:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No chapters found.")
+            else:
+                for chapter in chapters:
+                    chhref = chapter['href'].split('&')[0] # strip rs_session
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chhref))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+
+    def _postUrlUP(self, url):
+        params = {}
+        if self.password:
+            params['username'] = self.username
+            params['password'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['accept.x'] = 1
+        params['accept.y'] = 1
+
+        excpt=None
+        for sleeptime in [0.5, 1.5, 4, 9]:
+            time.sleep(sleeptime)	
+            try:
+                data = self._postUrl(url, params)
+                if data == "Unable to connect to the database":
+                    raise exceptions.FailedToDownload("Site reported 'Unable to connect to the database'")
+                if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data:
+                    raise exceptions.FailedToLogin(url,params['username'])
+                return data
+            except exceptions.FailedToLogin, ftl:
+                # no need to retry these.
+                raise(ftl)
+            except Exception, e:
+                excpt=e
+                logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
+
+        logger.error("Giving up on %s" %url)
+        logger.exception(excpt)
+        raise(excpt)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        data = self._postUrlUP(url)
+        #print("data:%s"%data)
+        
+        # some stories have html that confuses the parser.  For story
+        # text we don't care about anything before '<table id="page"'
+        # and seems to clear the issue.
+        data = data[data.index('<table id="page"'):]
+        
+        soup = bs.BeautifulSoup(data)
+        
+        div = soup.find('td',{'id':'page_content'})
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        ## Remove stuff from page_content
+
+        # Remove all tags before the first <hr> after class=info table (including hr)
+        hr = div.find('table',{'class':'info'}).findNext('hr')
+        for tag in hr.findAllPrevious():
+            tag.extract()
+        hr.extract()
+        
+        # Remove all tags after the last <hr> (including hr)
+        hr = div.findAll('hr')[-1]
+        for tag in hr.findAllNext():
+            tag.extract()
+        hr.extract()
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,11 +56,6 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','sda')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Supernatural")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y"
@@ -73,6 +69,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.samdean.archive.nu','samdean.archive.nu']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -85,7 +82,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -113,7 +110,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])
 
         # Find the chapters:
         chapters=soup.find('select')
@@ -211,6 +208,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -221,7 +219,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_scarheadnet.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ScarHeadNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ScarHeadNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','shn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'scarhead.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('tr',{'valign':'top'})
+        
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            if '/' == cat.string[0]:
+                self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0])
+            elif 'Harry' in cat.string:
+                self.story.addToList('ships',cat.string.split('(')[0])
+            else:
+                self.story.addToList('category',cat.string)
+            if '(' in cat.string:
+                self.story.addToList('category',cat.string.split('(')[1].split(')')[0])
+            
+                
+            
+
+        chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+
+        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+            
+        textsoup = stripHTML(soup)
+
+        a = textsoup.split('Published: ')[1].split(' ')[0]
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Updated: ')[1].split(' ')[0]
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Rating: ')[1].split(' ')[0]
+        self.story.setMetadata('rating', a)
+        a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0]
+        self.story.setMetadata('numWords', a)
+        a = textsoup.split('Completed: ')[1].split(' ')[0]
+        if 'Yes' in a:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        #a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0]
+        #self.setDescription(url,a)
+        
+
+        
+        a=soup.find(text=re.compile("Summary: "))
+        i=0
+        svalue = ""
+        while i == 0:
+            try:
+                b = str(a)
+                svalue += b.split('Summary: ')[1]
+            except:
+                svalue += str(a)
+            if a.nextSibling != None:
+                a = a.nextSibling
+            else:
+                a = a.parent.nextSibling
+            if 'Disclaimer: ' in stripHTML(a):
+                i=1
+        self.setDescription(url,svalue)
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,7 +48,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,13 +56,6 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','scacf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Glee")
-        self.story.addToList("characters","Kurt Hummel")
-        self.story.addToList("characters","Blaine Anderson")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"
@@ -71,6 +65,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.scarvesandcoffee.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -101,7 +96,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -121,7 +116,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -144,10 +139,10 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
@@ -228,6 +223,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -238,7 +234,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -47,19 +48,19 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        self.section=self.parsedUrl.path.split('/',)[1]
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/'+self.story.getMetadata('section')+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','sghp')
 
         # If all stories from the site fall into the same category,
         # the site itself isn't likely to label them as such, so we
-        # do.
-        if 'atlantis' in self.story.getMetadata('section'):
+        # do.  Can't use extracategories, could be Atlantis or SG-1
+        if 'atlantis' in self.section:
             self.story.addToList("category","Stargate: Atlantis")
         else:
             self.story.addToList("category","Stargate: SG-1")
@@ -73,6 +74,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'sg1-heliopolis.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
 
@@ -95,7 +97,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         url = self.url+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -115,7 +117,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 # correct stupid & error in url.
                 addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
-                logging.debug("URL 2nd try: "+url)
+                logger.debug("URL 2nd try: "+url)
 
                 try:
                     data = self._fetchUrl(url)
@@ -138,7 +140,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -149,7 +151,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -220,7 +222,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -232,6 +234,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -242,7 +245,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -0,0 +1,219 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SimplyUndeniableComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SimplyUndeniableComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sud')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.simplyundeniable.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Please log in now' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        if "Please log in now" in data:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: You need to have access to the restricted section.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for info in asoup.findAll('table', {'cellpadding' : '5'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',stripHTML(a))
+                break
+                
+        # Find the chapters:
+        if "Disclaimer" in data:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')):
+            # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        a = info.find('td', {'valign' : 'top'}).find('p')
+        self.setDescription(url,a)
+        
+        a = info.find('td', {'class' : 'greysm'}).findAll('b')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a[0].nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].nextSibling), self.dateformat))
+        if 'Yes' in a[2].nextSibling:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        self.story.setMetadata('rating', a[3].nextSibling)
+        self.story.setMetadata('numWords', a[4].nextSibling)
+        
+        
+        warnings = info.find('td', {'width' : '45'}).nextSibling.nextSibling.text.split(', ')
+        for warning in warnings:
+            if 'none' not in warning:
+                self.story.addToList('warnings',warning)
+        chars = info.find('td', {'width' : '51'}).nextSibling.nextSibling.text.split(', ')
+        for char in chars:
+            if '&' in char:
+                self.story.addToList('ships',char)
+            else:
+                self.story.addToList('characters',char)
+        genres = info.find('td', {'width' : '36'}).nextSibling.nextSibling.text.split(', ')
+        for genre in genres:
+            self.story.addToList('genre',genre)
+            
+        cat = info.find('a', href=re.compile(r'categories.php'))
+        self.story.addToList('category',cat.string)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('span', {'class' : 'style'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        div.name='div'
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -0,0 +1,251 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SinfulDesireOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SinfulDesireOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','snds')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.sinful-desire.org'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -48,7 +49,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -56,11 +57,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','siye') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d" # XXX
@@ -74,6 +70,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
     def getAcceptDomains(cls):
         return ['www.siye.co.uk','siye.co.uk']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
 
@@ -87,7 +84,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         # sites skip that for one-chapter stories.
         # Except it doesn't this time. :-/
         url = self.url #+'&index=1'+addurl
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -114,7 +111,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
 
         ## Title
         titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',titlea.string)
+        self.story.setMetadata('title',stripHTML(titlea))
         
         # Find the chapters (from soup, not authsoup):
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -219,6 +216,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -229,7 +227,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         # soup = bs.BeautifulSoup(self._fetchUrl(url))
         # BeautifulSoup objects to <p> inside <span>, which
@@ -243,5 +241,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         if None == story:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        story.name='div'
     
         return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_spikeluvercom.py

@@ -0,0 +1,206 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return SpikeluverComAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class SpikeluverComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'slc'
+    SITE_DOMAIN = 'spikeluver.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/SpuffyRealm/'
+    LOGIN_URL = BASE_URL + 'user.php?action=login'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d'
+    METADATA_URL_SUFFIX = '&index=1'
+    AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=5'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    STORY_DOES_NOT_EXIST_ERROR_TEXT = 'That story does not exist on this archive.  You may search for it or return to the home page.'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return SpikeluverComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX)
+
+        errortext_div = soup.find('div', {'class': 'errortext'})
+        if errortext_div:
+            error_text = ''.join(errortext_div(text=True)).strip()
+            if error_text == self.STORY_DOES_NOT_EXIST_ERROR_TEXT:
+                raise exceptions.StoryDoesNotExist(self.url)
+
+        # No additional login is required, just check for adult
+        pagetitle_div = soup.find('div', id='pagetitle')
+        if pagetitle_div.a['href'].startswith('javascript:'):
+            if not(self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+        url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
+        soup = self._customized_fetch_url(url)
+
+        pagetitle_div = soup.find('div', id='pagetitle')
+        self.story.setMetadata('title', pagetitle_div.a.string.strip())
+
+        author_anchor = pagetitle_div.a.findNextSibling('a')
+        url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        components = urlparse.urlparse(url)
+        query_data = urlparse.parse_qs(components.query)
+
+        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', url)
+
+        sort_div = soup.find('div', id='sort')
+        self.story.setMetadata('reviews', sort_div('a')[1].string.strip())
+
+        listbox_tag = soup.find('div', {'class': 'listbox'})
+        for span_tag in listbox_tag('span'):
+            key = span_tag.string.strip(' :')
+            try:
+                value = span_tag.nextSibling.string.strip()
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(span_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        # Encountered next label, break. Not as bad as other
+                        # e-fiction sites, let's hope this is enough for proper
+                        # parsing.
+                        if sibling.name == 'span' and sibling.get('class', None) == 'label':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(self.url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Remove the preceding break line tag and other crud
+                contents.pop()
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Rated':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Categories':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+
+                    self.story.addToList('category', sibling.string.strip())
+
+            # Seems to be always "None" for some reason
+            elif key == 'Characters':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('characters', sibling.string.strip())
+
+            elif key == 'Genres':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+
+                    self.story.addToList('genre', sibling.string.strip())
+
+            elif key == 'Warnings':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('warnings', sibling.string.strip())
+
+            # Challenges
+
+            elif key == 'Series':
+                a = span_tag.findNextSibling('a')
+                if not a:
+                    continue
+                self.story.setMetadata('series', a.string.strip())
+                self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Word count':
+                self.story.setMetadata('numWords', value)
+
+            elif key == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+        for p_tag in listbox_tag.findNextSiblings('p'):
+            chapter_anchor = p_tag.find('a', href=lambda href: href and href.startswith('viewstory.php?sid='))
+            if not chapter_anchor:
+                continue
+
+            title = chapter_anchor.string.strip()
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((title, url))
+
+    def getChapterText(self, url):
+        url += self.AGE_CONSENT_URL_SUFFIX
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -17,6 +17,7 @@
 
 import time
 import logging
+logger = logging.getLogger(__name__)
 import re
 import urllib2
 
@@ -58,10 +59,10 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('https://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','wwomb')
@@ -75,22 +76,23 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.squidge.org'
     
-    @classmethod # must be @staticmethod, don't remove it.
+    @classmethod # must be @classmethod, don't remove it.
     def getConfigSection(cls):
         # The config section name.  Only override if != site domain.
         return cls.getSiteDomain()+'/peja'
     
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
+        return "https://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/")+"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
+        return r"https?"+re.escape("://"+self.getSiteDomain()+"/")+r"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url
-        logging.debug("URL: "+url)
+        logger.debug("URL: "+url)
 
         try:
             data = self._fetchUrl(url)
@@ -101,6 +103,9 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
                 raise e
 
         data = self._fetchUrl(url)
+
+        if "fatal MySQL error was encountered" in data:
+            raise exceptions.FailedToDownload("Site SQL Error--bad story")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -111,7 +116,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         author = soup.find('div', {'id':"pagetitle"}).find('a')
         self.story.setMetadata('authorId',author['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/peja/cgi-bin/'+author['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/peja/cgi-bin/'+author['href'])
         self.story.setMetadata('author',author.string)
 		
         authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
@@ -120,13 +125,13 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # formating, so it's a little ugly.
 
         title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')})
-        self.story.setMetadata('title',title.string)
+        self.story.setMetadata('title',stripHTML(title))
         titleblock=title.parent.parent
         
         chapterselect=soup.find('select',{'name':'chapter'})
         if chapterselect:
             for ch in chapterselect.findAll('option'):
-                self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
+                self.chapterUrls.append((stripHTML(ch),'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
         else:
             self.chapterUrls.append((title,url))
 		
@@ -156,6 +161,8 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
                 #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
+                if value.endswith("["):
+                    value = value[:-1]
                 self.story.setMetadata('rating', value)
 
             if 'Characters' in label:
@@ -200,7 +207,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             # http://www.squidge.org/peja/cgi-bin/series.php?seriesid=254
             a = titleblock.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/peja/cgi-bin/'+a['href']
+            series_url = 'https://'+self.host+'/peja/cgi-bin/'+a['href']
     
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -209,6 +216,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
                 
@@ -221,7 +229,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
-        logging.debug('Getting chapter text from: %s' % url)
+        logger.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.