Change calls for str() to unicode(). Default CL type to epub if not given.

the from __future__ import with_statement needs to be the first import statment as well.

BeautifulSoup.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 """Beautiful Soup
 Elixir and Tonic
 "The Screen-Scraper's Friend"
@@ -79,7 +81,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT.
 from __future__ import generators
 
 __author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "3.2.0"
+__version__ = "3.0.8.1"
 __copyright__ = "Copyright (c) 2004-2010 Leonard Richardson"
 __license__ = "New-style BSD"
 
@@ -531,8 +533,6 @@ class Tag(PageElement):
         self.name = name
         if attrs is None:
             attrs = []
-        elif isinstance(attrs, dict):
-            attrs = attrs.items()
         self.attrs = attrs
         self.contents = []
         self.setup(parent, previous)

__init__.py

@@ -1 +1 @@
-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-

adapter.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+
+import logging
+import datetime
+from output import makeAcceptableFilename
+
+try:
+	from google.appengine.api.urlfetch import fetch as googlefetch
+	appEngineGlob = True
+except:
+	appEngineGlob = False
+
+class LoginRequiredException(Exception):
+	def __init__(self, url):
+		self.url = url
+	
+	def __str__(self):
+		return repr(self.url + ' requires user to be logged in')
+	
+class StoryArchivedAlready(Exception):
+	pass
+
+class StoryDoesNotExist(Exception):
+	pass
+
+class FailedToDownload(Exception):
+	pass
+
+class InvalidStoryURL(Exception):
+	pass
+
+class FanfictionSiteAdapter:
+	appEngine = appEngineGlob
+	login = ''
+	password = ''
+	url = ''
+	host = ''
+	path = ''
+	uuid = ''
+	storyName = ''
+	storyId = ''
+	authorName = ''
+	authorId = ''
+	authorURL = ''
+	outputStorySep = '-Ukn_'
+	outputName = ''
+	outputFileName = ''
+	storyDescription = ''
+	storyCharacters = []
+	storySeries = ''
+	storyPublished = datetime.date(1970, 01, 31)
+	storyCreated = datetime.datetime.now()
+	storyUpdated = datetime.date(1970, 01, 31)
+	languageId = 'en-UK'
+	language = 'English'
+	subjects = []
+	publisher = ''
+	numChapters = '0'
+	numWords = '0'
+	genre = ''
+	category = ''
+	storyStatus = 'In-Progress'
+	storyRating = ''
+	storyUserRating = '0'
+	def __init__(self, url):
+		# basic plain url parsing...
+		self.url = url
+		parsedUrl = up.urlparse(url)
+		self.host = parsedUrl.netloc
+		self.path = parsedUrl.path
+			
+	def hasAppEngine(self):
+		return self.appEngine
+	
+	def fetchUrl(self, url):
+		if not self.appEngine:
+			return self.opener.open(url).read().decode('utf-8')
+		else:
+			return googlefetch(url).content
+	
+	def requiresLogin(self, url = None):
+		return False
+	
+	def performLogin(self, url = None):
+		return True
+	
+	def extractIndividualUrls(self):
+		pass
+		
+	def getText(self, url):
+		pass
+
+	def setLogin(self, login):
+		self.login = login
+
+	def setPassword(self, password):
+		self.password = password
+
+	def getHost(self):
+		logging.debug('self.host=%s' % self.host)
+		return self.host
+	
+	def getUUID(self):
+		self.uuid = 'urn:uuid:' + self.host + '-u.' + self.authorId + '-s.' + self.storyId
+		logging.debug('self.uuid=%s' % self.uuid)
+		return self.uuid
+
+	def getOutputName(self):
+		self.outputName = makeAcceptableFilename(self.storyName.replace(" ", "_") + self.outputStorySep + self.storyId)
+		logging.debug('self.outputName=%s' % self.outputName)
+		return self.outputName
+
+	def getOutputFileName(self, booksDirectory, bookExt):
+		self.getOutputName()	# make sure self.outputName is populated
+		self.outputFileName = booksDirectory + "/" + self.outputName + bookExt
+		logging.debug('self.outputFileName=%s' % self.outputFileName)
+		return self.outputFileName
+
+	def getAuthorURL(self):
+		logging.debug('self.authorURL=%s' % self.authorURL)
+		return self.authorURL
+
+	def getAuthorId(self):
+		logging.debug('self.authorId=%s' % self.authorId)
+		return self.authorId
+
+	def getAuthorName(self):
+		logging.debug('self.authorName=%s' % self.authorName)
+		return self.authorName
+
+	def getStoryURL(self):
+		logging.debug('self.url=%s' % self.url)
+		return self.url
+
+	def getStoryId(self):
+		logging.debug('self.storyId=%s' % self.storyId)
+		return self.storyId
+
+	def getStoryName(self):
+		logging.debug('self.storyName=%s' % self.storyName)
+		return self.storyName
+
+	def getStoryDescription(self):
+		logging.debug('self.storyDescription=%s' % self.storyDescription)
+		return self.storyDescription
+
+	def getStoryCreated(self):
+		self.storyCreated = datetime.datetime.now()
+		logging.debug('self.storyCreated=%s' % self.storyCreated)
+		return self.storyCreated
+
+	def addCharacter(self, character):
+		chara = character.upper()
+		for c in self.storyCharacters:
+			if c.upper() == chara:
+				return False
+		self.storyCharacters.append(character)
+		return True
+
+	def getStoryCharacters(self):
+		logging.debug('self.storyCharacters=%s' % self.storyCharacters)
+		return self.storyCharacters
+	
+	def getStoryPublished(self):
+		logging.debug('self.storyPublished=%s' % self.storyPublished)
+		return self.storyPublished
+
+	def getStoryUpdated(self):
+		logging.debug('self.storyUpdated=%s' % self.storyUpdated)
+		return self.storyUpdated
+
+	def getStorySeries(self):
+		logging.debug('self.storySeries=%s' % self.storySeries)
+		return self.storySeries
+
+	def getLanguage(self):
+		logging.debug('self.language=%s' % self.language)
+		return self.language
+
+	def getLanguageId(self):
+		logging.debug('self.languageId=%s' % self.languageId)
+		return self.languageId
+
+	def addSubject(self, subject):
+		subj = subject.upper()
+		for s in self.subjects:
+			if s.upper() == subj:
+				return False
+		self.subjects.append(subject)
+		return True
+
+	def getSubjects(self):
+		logging.debug('self.subjects=%s' % self.subjects)
+		return self.subjects
+
+	def getPublisher(self):
+		logging.debug('self.publisher=%s' % self.publisher)
+		return self.publisher
+
+	def getNumChapters(self):
+		logging.debug('self.numChapters=%s' % self.numChapters)
+		return self.numChapters
+
+	def getNumWords(self):
+		logging.debug('self.numWords=%s' % self.numWords)
+		return self.numWords
+
+	def getCategory(self):
+		logging.debug('self.category=%s' % self.category)
+		return self.category
+
+	def getGenre(self):
+		logging.debug('self.genre=%s' % self.genre)
+		return self.genre
+
+	def getStoryStatus(self):
+		logging.debug('self.storyStatus=%s' % self.storyStatus)
+		return self.storyStatus
+
+	def getStoryRating(self):
+		logging.debug('self.storyRating=%s' % self.storyRating)
+		return self.storyRating
+
+	def getStoryUserRating(self):
+		logging.debug('self.storyUserRating=%s' % self.storyUserRating)
+		return self.storyUserRating
+
+	def getPrintableUrl(self, url):
+		return url

allrecent.html

@@ -1,78 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">
-<html>
-  <head>
-    <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
-    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-    <script type="text/javascript">
-
-      var _gaq = _gaq || [];
-      _gaq.push(['_setAccount', 'UA-12136939-1']);
-      _gaq.push(['_trackPageview']);
-
-      (function() {
-      var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
-      ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
-      var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
-      })();
-
-    </script>
-  </head>
-  <body>
-    <div id='main'>
-      <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
-      </h1>
-      
-      <script type="text/javascript"><!--
-                                        google_ad_client = "ca-pub-0320924304307555";
-                                        /* Standard */
-                                        google_ad_slot = "8974025478";
-                                        google_ad_width = 468;
-                                        google_ad_height = 60;
-                                        //-->
-      </script>
-      <script type="text/javascript"
-              src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
-      </script>
-      <!--			<div id='yourfile'> -->
-      {{yourfile}}
-      <!--			</div> -->
-      
-      <div id='helpbox'>
-	{% for fic in fics %}
-	<p>
-  	  <a href="{{ fic.url }}" title="Link to original story"><span class="recent"><i>{{ fic.title }}</i></span></a>
-	      by <a href="{{ fic.authorUrl }}">{{ fic.author }}</a>  <b>Download Count:</b> {{ fic.count }} <br />
-	  <b>Word Count:</b> {{ fic.numWords }} <b>Chapter Count:</b> {{ fic.numChapters }}<br />
-	  {% if fic.category %} <b>Categories:</b> {{ fic.category }} <br /> {% endif %}
-	  {% if fic.genre %} <b>Genres:</b> {{ fic.genre }} <br /> {% endif %}
-	  {% if fic.language %} <b>Language:</b> {{ fic.language }} <br /> {% endif %}
-	  {% if fic.series %} <b>Series:</b> {{ fic.series }} <br /> {% endif %}
-	  {% if fic.characters %} <b>Characters:</b> {{ fic.characters }} <br /> {% endif %}
-	  {% if fic.status %} <b>Status:</b> {{ fic.status }} <br /> {% endif %}
-	  {% if fic.datePublished %} <b>Published:</b> {{ fic.datePublished }} <br /> {% endif %}
-	  {% if fic.dateUpdated %} <b>Last Updated:</b> {{ fic.dateUpdated }} <br /> {% endif %}
-	  {% if fic.dateCreated %} <b>Last Downloaded:</b> {{ fic.dateCreated }} <br /> {% endif %}
-	  {% if fic.rating %} <b>Rating:</b> {{ fic.rating }} <br /> {% endif %}
-	  {% if fic.warnings %} <b>Warnings:</b> {{ fic.warnings }} <br /> {% endif %}
-	  {% if fic.description %} <b>Summary:</b> {{ fic.description }} <br /> {% endif %}
-	</p>
-	{% endfor %}
-      </div>
-      
-      <script type="text/javascript"><!--
-                                        google_ad_client = "ca-pub-0320924304307555";
-                                        /* Standard */
-                                        google_ad_slot = "8974025478";
-                                        google_ad_width = 468;
-                                        google_ad_height = 60;
-                                        //-->
-      </script>
-      <script type="text/javascript"
-              src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
-      </script>				
-      
-    </div>
-  </body>
-</html>

app.yaml

@@ -1,46 +0,0 @@
-# ffd-retief-hrd fanfictiondownloader
-application: fanfictiondownloader
-version: 4-4-29
-runtime: python27
-api_version: 1
-threadsafe: true
-
-handlers:
-
-- url: /r3m0v3r.*
-  script: utils.remover.app
-  login: admin
-
-- url: /tally.*
-  script: utils.tally.app
-  login: admin
-
-- url: /fdownloadtask
-  script: main.app
-  login: admin
-  
-- url: /css
-  static_dir: css
-
-- url: /js
-  static_dir: js
-
-- url: /static
-  static_dir: static
-
-- url: /favicon\.ico
-  static_files: static/favicon.ico
-  upload: static/favicon\.ico
-
-- url: /.*
-  script: main.app
-
-#builtins:
-#- datastore_admin: on
-
-libraries:
-- name: django
-  version: "1.2"
-
-- name: PIL
-  version: "1.1.7"

calibre-plugin/__init__.py

@@ -1,90 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-# -*- coding: utf-8 -*-
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
-__docformat__ = 'restructuredtext en'
-
-# The class that all Interface Action plugin wrappers must inherit from
-from calibre.customize import InterfaceActionBase
-
-## Apparently the name for this class doesn't matter--it was still
-## 'demo' for the first few versions.
-class FanFictionDownLoaderBase(InterfaceActionBase):
-    '''
-    This class is a simple wrapper that provides information about the
-    actual plugin class. The actual interface plugin class is called
-    InterfacePlugin and is defined in the ffdl_plugin.py file, as
-    specified in the actual_plugin field below.
-
-    The reason for having two classes is that it allows the command line
-    calibre utilities to run without needing to load the GUI libraries.
-    '''
-    name                = 'FanFictionDownLoader'
-    description         = 'UI plugin to download FanFiction stories from various sites.'
-    supported_platforms = ['windows', 'osx', 'linux']
-    author              = 'Jim Miller'
-    version             = (1, 6, 14)
-    minimum_calibre_version = (0, 8, 57)
-
-    #: This field defines the GUI plugin class that contains all the code
-    #: that actually does something. Its format is module_path:class_name
-    #: The specified class must be defined in the specified module.
-    actual_plugin       = 'calibre_plugins.fanfictiondownloader_plugin.ffdl_plugin:FanFictionDownLoaderPlugin'
-
-    def is_customizable(self):
-        '''
-        This method must return True to enable customization via
-        Preferences->Plugins
-        '''
-        return True
-
-    def config_widget(self):
-        '''
-        Implement this method and :meth:`save_settings` in your plugin to
-        use a custom configuration dialog.
-
-        This method, if implemented, must return a QWidget. The widget can have
-        an optional method validate() that takes no arguments and is called
-        immediately after the user clicks OK. Changes are applied if and only
-        if the method returns True.
-
-        If for some reason you cannot perform the configuration at this time,
-        return a tuple of two strings (message, details), these will be
-        displayed as a warning dialog to the user and the process will be
-        aborted.
-
-        The base class implementation of this method raises NotImplementedError
-        so by default no user configuration is possible.
-        '''
-        # It is important to put this import statement here rather than at the
-        # top of the module as importing the config class will also cause the
-        # GUI libraries to be loaded, which we do not want when using calibre
-        # from the command line
-        from calibre_plugins.fanfictiondownloader_plugin.config import ConfigWidget
-        return ConfigWidget(self.actual_plugin_)
-
-    def save_settings(self, config_widget):
-        '''
-        Save the settings specified by the user with config_widget.
-
-        :param config_widget: The widget returned by :meth:`config_widget`.
-        '''
-        config_widget.save_settings()
-
-        # Apply the changes
-        ac = self.actual_plugin_
-        if ac is not None:
-            ac.apply_settings()
-
-# For testing, run from command line with this:
-# calibre-debug -e __init__.py
-# 
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    from calibre.gui2.preferences import test_widget
-    app = QApplication([])
-    test_widget('Advanced', 'Plugins')

calibre-plugin/about.txt

@@ -1,28 +0,0 @@
-<hr />
-
-<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
-'<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
-'<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
-'<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.  bbcodeutils code contributed by Pau Sanchez.</p>
-
-<p>
-Calibre officially distributes plugins from the mobileread.com forum site.
-The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
-</p>
-
-<p> I also monitor the
-<a href="http://groups.google.com/group/fanfic-downloader">general users
-group</a> for the downloader.  That covers the web application and CLI, too.
-</p>
-
-The source for this plugin is available at it's 
-<a href="http://code.google.com/p/fanficdownloader">project home</a>.
-<hr />
-
-<p>
-See the <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">list of supported sites</a>.
-</p>
-<p>
-Read the <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
-</p>

calibre-plugin/common_utils.py

@@ -1,543 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
-__docformat__ = 'restructuredtext en'
-
-import os
-from PyQt4 import QtGui
-from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                      QTextEdit,
-                      QListWidget, QAbstractItemView)
-from calibre.constants import iswindows
-from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
-from calibre.gui2.actions import menu_action_unique_name
-from calibre.gui2.keyboard import ShortcutConfig
-from calibre.utils.config import config_dir
-from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE
-
-# Global definition of our plugin name. Used for common functions that require this.
-plugin_name = None
-# Global definition of our plugin resources. Used to share between the xxxAction and xxxBase
-# classes if you need any zip images to be displayed on the configuration dialog.
-plugin_icon_resources = {}
-
-
-def set_plugin_icon_resources(name, resources):
-    '''
-    Set our global store of plugin name and icon resources for sharing between
-    the InterfaceAction class which reads them and the ConfigWidget
-    if needed for use on the customization dialog for this plugin.
-    '''
-    global plugin_icon_resources, plugin_name
-    plugin_name = name
-    plugin_icon_resources = resources
-
-
-def get_icon(icon_name):
-    '''
-    Retrieve a QIcon for the named image from the zip file if it exists,
-    or if not then from Calibre's image cache.
-    '''
-    if icon_name:
-        pixmap = get_pixmap(icon_name)
-        if pixmap is None:
-            # Look in Calibre's cache for the icon
-            return QIcon(I(icon_name))
-        else:
-            return QIcon(pixmap)
-    return QIcon()
-
-
-def get_pixmap(icon_name):
-    '''
-    Retrieve a QPixmap for the named image
-    Any icons belonging to the plugin must be prefixed with 'images/'
-    '''
-    global plugin_icon_resources, plugin_name
-
-    if not icon_name.startswith('images/'):
-        # We know this is definitely not an icon belonging to this plugin
-        pixmap = QPixmap()
-        pixmap.load(I(icon_name))
-        return pixmap
-
-    # Check to see whether the icon exists as a Calibre resource
-    # This will enable skinning if the user stores icons within a folder like:
-    # ...\AppData\Roaming\calibre\resources\images\Plugin Name\
-    if plugin_name:
-        local_images_dir = get_local_images_dir(plugin_name)
-        local_image_path = os.path.join(local_images_dir, icon_name.replace('images/', ''))
-        if os.path.exists(local_image_path):
-            pixmap = QPixmap()
-            pixmap.load(local_image_path)
-            return pixmap
-
-    # As we did not find an icon elsewhere, look within our zip resources
-    if icon_name in plugin_icon_resources:
-        pixmap = QPixmap()
-        pixmap.loadFromData(plugin_icon_resources[icon_name])
-        return pixmap
-    return None
-
-
-def get_local_images_dir(subfolder=None):
-    '''
-    Returns a path to the user's local resources/images folder
-    If a subfolder name parameter is specified, appends this to the path
-    '''
-    images_dir = os.path.join(config_dir, 'resources/images')
-    if subfolder:
-        images_dir = os.path.join(images_dir, subfolder)
-    if iswindows:
-        images_dir = os.path.normpath(images_dir)
-    return images_dir
-
-
-def create_menu_item(ia, parent_menu, menu_text, image=None, tooltip=None,
-                     shortcut=(), triggered=None, is_checked=None):
-    '''
-    Create a menu action with the specified criteria and action
-    Note that if no shortcut is specified, will not appear in Preferences->Keyboard
-    This method should only be used for actions which either have no shortcuts,
-    or register their menus only once. Use create_menu_action_unique for all else.
-    '''
-    if shortcut is not None:
-        if len(shortcut) == 0:
-            shortcut = ()
-        else:
-            shortcut = _(shortcut)
-    ac = ia.create_action(spec=(menu_text, None, tooltip, shortcut),
-        attr=menu_text)
-    if image:
-        ac.setIcon(get_icon(image))
-    if triggered is not None:
-        ac.triggered.connect(triggered)
-    if is_checked is not None:
-        ac.setCheckable(True)
-        if is_checked:
-            ac.setChecked(True)
-
-    parent_menu.addAction(ac)
-    return ac
-
-
-def create_menu_action_unique(ia, parent_menu, menu_text, image=None, tooltip=None,
-                       shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
-                       unique_name=None):
-    '''
-    Create a menu action with the specified criteria and action, using the new
-    InterfaceAction.create_menu_action() function which ensures that regardless of
-    whether a shortcut is specified it will appear in Preferences->Keyboard
-    '''
-    orig_shortcut = shortcut
-    kb = ia.gui.keyboard
-    if unique_name is None:
-        unique_name = menu_text
-    if not shortcut == False:
-        full_unique_name = menu_action_unique_name(ia, unique_name)
-        if full_unique_name in kb.shortcuts:
-            shortcut = False
-        else:
-            if shortcut is not None and not shortcut == False:
-                if len(shortcut) == 0:
-                    shortcut = None
-                else:
-                    shortcut = _(shortcut)
-
-    if shortcut_name is None:
-        shortcut_name = menu_text.replace('&','')
-
-    ac = ia.create_menu_action(parent_menu, unique_name, menu_text, icon=None, shortcut=shortcut,
-        description=tooltip, triggered=triggered, shortcut_name=shortcut_name)
-    if shortcut == False and not orig_shortcut == False:
-        if ac.calibre_shortcut_unique_name in ia.gui.keyboard.shortcuts:
-            kb.replace_action(ac.calibre_shortcut_unique_name, ac)
-    if image:
-        ac.setIcon(get_icon(image))
-    if is_checked is not None:
-        ac.setCheckable(True)
-        if is_checked:
-            ac.setChecked(True)
-    return ac
-
-
-def swap_author_names(author):
-    if author.find(',') == -1:
-        return author
-    name_parts = author.strip().partition(',')
-    return name_parts[2].strip() + ' ' + name_parts[0]
-
-
-def get_library_uuid(db):
-    try:
-        library_uuid = db.library_id
-    except:
-        library_uuid = ''
-    return library_uuid
-
-
-class ImageLabel(QLabel):
-
-    def __init__(self, parent, icon_name, size=16):
-        QLabel.__init__(self, parent)
-        pixmap = get_pixmap(icon_name)
-        self.setPixmap(pixmap)
-        self.setMaximumSize(size, size)
-        self.setScaledContents(True)
-
-
-class ImageTitleLayout(QHBoxLayout):
-    '''
-    A reusable layout widget displaying an image followed by a title
-    '''
-    def __init__(self, parent, icon_name, title):
-        QHBoxLayout.__init__(self)
-        title_image_label = QLabel(parent)
-        pixmap = get_pixmap(icon_name)
-        if pixmap is None:
-            pixmap = get_pixmap('library.png')
-            # error_dialog(parent, _('Restart required'),
-            #              _('You must restart Calibre before using this plugin!'), show=True)
-        else:
-            title_image_label.setPixmap(pixmap)
-        title_image_label.setMaximumSize(32, 32)
-        title_image_label.setScaledContents(True)
-        self.addWidget(title_image_label)
-
-        title_font = QFont()
-        title_font.setPointSize(16)
-        shelf_label = QLabel(title, parent)
-        shelf_label.setFont(title_font)
-        self.addWidget(shelf_label)
-        self.insertStretch(-1)
-
-
-class SizePersistedDialog(QDialog):
-    '''
-    This dialog is a base class for any dialogs that want their size/position
-    restored when they are next opened.
-    '''
-    def __init__(self, parent, unique_pref_name):
-        QDialog.__init__(self, parent)
-        self.unique_pref_name = unique_pref_name
-        self.geom = gprefs.get(unique_pref_name, None)
-        self.finished.connect(self.dialog_closing)
-
-    def resize_dialog(self):
-        if self.geom is None:
-            self.resize(self.sizeHint())
-        else:
-            self.restoreGeometry(self.geom)
-
-    def dialog_closing(self, result):
-        geom = bytearray(self.saveGeometry())
-        gprefs[self.unique_pref_name] = geom
-
-
-class ReadOnlyTableWidgetItem(QTableWidgetItem):
-
-    def __init__(self, text):
-        if text is None:
-            text = ''
-        QTableWidgetItem.__init__(self, text, QtGui.QTableWidgetItem.UserType)
-        self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
-
-
-class RatingTableWidgetItem(QTableWidgetItem):
-
-    def __init__(self, rating, is_read_only=False):
-        QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
-        self.setData(Qt.DisplayRole, rating)
-        if is_read_only:
-            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
-
-
-class DateTableWidgetItem(QTableWidgetItem):
-
-    def __init__(self, date_read, is_read_only=False, default_to_today=False):
-        if date_read == UNDEFINED_DATE and default_to_today:
-            date_read = now()
-        if is_read_only:
-            QTableWidgetItem.__init__(self, format_date(date_read, None), QtGui.QTableWidgetItem.UserType)
-            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
-        else:
-            QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
-            self.setData(Qt.DisplayRole, QDateTime(date_read))
-
-
-class NoWheelComboBox(QComboBox):
-
-    def wheelEvent (self, event):
-        # Disable the mouse wheel on top of the combo box changing selection as plays havoc in a grid
-        event.ignore()
-
-
-class CheckableTableWidgetItem(QTableWidgetItem):
-
-    def __init__(self, checked=False, is_tristate=False):
-        QTableWidgetItem.__init__(self, '')
-        self.setFlags(Qt.ItemFlags(Qt.ItemIsSelectable | Qt.ItemIsUserCheckable | Qt.ItemIsEnabled ))
-        if is_tristate:
-            self.setFlags(self.flags() | Qt.ItemIsTristate)
-        if checked:
-            self.setCheckState(Qt.Checked)
-        else:
-            if is_tristate and checked is None:
-                self.setCheckState(Qt.PartiallyChecked)
-            else:
-                self.setCheckState(Qt.Unchecked)
-
-    def get_boolean_value(self):
-        '''
-        Return a boolean value indicating whether checkbox is checked
-        If this is a tristate checkbox, a partially checked value is returned as None
-        '''
-        if self.checkState() == Qt.PartiallyChecked:
-            return None
-        else:
-            return self.checkState() == Qt.Checked
-
-
-class TextIconWidgetItem(QTableWidgetItem):
-
-    def __init__(self, text, icon):
-        QTableWidgetItem.__init__(self, text)
-        if icon:
-            self.setIcon(icon)
-
-
-class ReadOnlyTextIconWidgetItem(ReadOnlyTableWidgetItem):
-
-    def __init__(self, text, icon):
-        ReadOnlyTableWidgetItem.__init__(self, text)
-        if icon:
-            self.setIcon(icon)
-
-
-class ReadOnlyLineEdit(QLineEdit):
-
-    def __init__(self, text, parent):
-        if text is None:
-            text = ''
-        QLineEdit.__init__(self, text, parent)
-        self.setEnabled(False)
-
-
-class KeyValueComboBox(QComboBox):
-
-    def __init__(self, parent, values, selected_key):
-        QComboBox.__init__(self, parent)
-        self.values = values
-        self.populate_combo(selected_key)
-
-    def populate_combo(self, selected_key):
-        self.clear()
-        selected_idx = idx = -1
-        for key, value in self.values.iteritems():
-            idx = idx + 1
-            self.addItem(value)
-            if key == selected_key:
-                selected_idx = idx
-        self.setCurrentIndex(selected_idx)
-
-    def selected_key(self):
-        for key, value in self.values.iteritems():
-            if value == unicode(self.currentText()).strip():
-                return key
-
-
-class CustomColumnComboBox(QComboBox):
-
-    def __init__(self, parent, custom_columns, selected_column, initial_items=['']):
-        QComboBox.__init__(self, parent)
-        self.populate_combo(custom_columns, selected_column, initial_items)
-
-    def populate_combo(self, custom_columns, selected_column, initial_items=['']):
-        self.clear()
-        self.column_names = initial_items
-        if len(initial_items) > 0:
-            self.addItems(initial_items)
-        selected_idx = 0
-        for idx, value in enumerate(initial_items):
-            if value == selected_column:
-                selected_idx = idx
-        for key in sorted(custom_columns.keys()):
-            self.column_names.append(key)
-            self.addItem('%s (%s)'%(key, custom_columns[key]['name']))
-            if key == selected_column:
-                selected_idx = len(self.column_names) - 1
-        self.setCurrentIndex(selected_idx)
-
-    def get_selected_column(self):
-        return self.column_names[self.currentIndex()]
-
-
-class KeyboardConfigDialog(SizePersistedDialog):
-    '''
-    This dialog is used to allow editing of keyboard shortcuts.
-    '''
-    def __init__(self, gui, group_name):
-        SizePersistedDialog.__init__(self, gui, 'Keyboard shortcut dialog')
-        self.gui = gui
-        self.setWindowTitle('Keyboard shortcuts')
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-
-        self.keyboard_widget = ShortcutConfig(self)
-        layout.addWidget(self.keyboard_widget)
-        self.group_name = group_name
-
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.commit)
-        button_box.rejected.connect(self.reject)
-        layout.addWidget(button_box)
-
-        # Cause our dialog size to be restored from prefs or created on first usage
-        self.resize_dialog()
-        self.initialize()
-
-    def initialize(self):
-        self.keyboard_widget.initialize(self.gui.keyboard)
-        self.keyboard_widget.highlight_group(self.group_name)
-
-    def commit(self):
-        self.keyboard_widget.commit()
-        self.accept()
-
-
-class DateDelegate(QStyledItemDelegate):
-    '''
-    Delegate for dates. Because this delegate stores the
-    format as an instance variable, a new instance must be created for each
-    column. This differs from all the other delegates.
-    '''
-    def __init__(self, parent):
-        QStyledItemDelegate.__init__(self, parent)
-        self.format = 'dd MMM yyyy'
-
-    def displayText(self, val, locale):
-        d = val.toDateTime()
-        if d <= UNDEFINED_QDATETIME:
-            return ''
-        return format_date(qt_to_dt(d, as_utc=False), self.format)
-
-    def createEditor(self, parent, option, index):
-        qde = QStyledItemDelegate.createEditor(self, parent, option, index)
-        qde.setDisplayFormat(self.format)
-        qde.setMinimumDateTime(UNDEFINED_QDATETIME)
-        qde.setSpecialValueText(_('Undefined'))
-        qde.setCalendarPopup(True)
-        return qde
-
-    def setEditorData(self, editor, index):
-        val = index.model().data(index, Qt.DisplayRole).toDateTime()
-        if val is None or val == UNDEFINED_QDATETIME:
-            val = now()
-        editor.setDateTime(val)
-
-    def setModelData(self, editor, model, index):
-        val = editor.dateTime()
-        if val <= UNDEFINED_QDATETIME:
-            model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
-        else:
-            model.setData(index, QDateTime(val), Qt.EditRole)
-            
-class PrefsViewerDialog(SizePersistedDialog):
-
-    def __init__(self, gui, namespace):
-        SizePersistedDialog.__init__(self, gui, 'Prefs Viewer dialog')
-        self.setWindowTitle('Preferences for: '+namespace)
-
-        self.gui = gui
-        self.db = gui.current_db
-        self.namespace = namespace
-        self._init_controls()
-        self.resize_dialog()
-
-        self._populate_settings()
-
-        if self.keys_list.count():
-            self.keys_list.setCurrentRow(0)
-
-    def _init_controls(self):
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-
-        ml = QHBoxLayout()
-        layout.addLayout(ml, 1)
-
-        self.keys_list = QListWidget(self)
-        self.keys_list.setSelectionMode(QAbstractItemView.SingleSelection)
-        self.keys_list.setFixedWidth(150)
-        self.keys_list.setAlternatingRowColors(True)
-        ml.addWidget(self.keys_list)
-        self.value_text = QTextEdit(self)
-        self.value_text.setTabStopWidth(24)
-        self.value_text.setReadOnly(True)
-        ml.addWidget(self.value_text, 1)
-
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok)
-        button_box.accepted.connect(self.accept)
-        self.clear_button = button_box.addButton('Clear', QDialogButtonBox.ResetRole)
-        self.clear_button.setIcon(get_icon('trash.png'))
-        self.clear_button.setToolTip('Clear all settings for this plugin')
-        self.clear_button.clicked.connect(self._clear_settings)
-        layout.addWidget(button_box)
-
-    def _populate_settings(self):
-        self.keys_list.clear()
-        ns_prefix = self._get_ns_prefix()
-        keys = sorted([k[len(ns_prefix):] for k in self.db.prefs.iterkeys()
-                       if k.startswith(ns_prefix)])
-        for key in keys:
-            self.keys_list.addItem(key)
-        self.keys_list.setMinimumWidth(self.keys_list.sizeHintForColumn(0))
-        self.keys_list.currentRowChanged[int].connect(self._current_row_changed)
-
-    def _current_row_changed(self, new_row):
-        if new_row < 0:
-            self.value_text.clear()
-            return
-        key = unicode(self.keys_list.currentItem().text())
-        val = self.db.prefs.get_namespaced(self.namespace, key, '')
-        self.value_text.setPlainText(self.db.prefs.to_raw(val))
-        
-    def _get_ns_prefix(self):
-        return 'namespaced:%s:'% self.namespace
-        
-    def _clear_settings(self):
-        from calibre.gui2.dialogs.confirm_delete import confirm
-        message = '<p>Are you sure you want to clear your settings in this library for this plugin?</p>' \
-                  '<p>Any settings in other libraries or stored in a JSON file in your calibre plugins ' \
-                  'folder will not be touched.</p>' \
-                  '<p>You must restart calibre afterwards.</p>'
-        if not confirm(message, self.namespace+'_clear_settings', self):
-            return
-        ns_prefix = self._get_ns_prefix()
-        keys = [k for k in self.db.prefs.iterkeys() if k.startswith(ns_prefix)]
-        for k in keys:
-            del self.db.prefs[k]
-        self._populate_settings()
-        d = info_dialog(self, 'Settings deleted', 
-                        '<p>All settings for this plugin in this library have been cleared.</p>'
-                        '<p>Please restart calibre now.</p>',
-                        show_copy_button=False)
-        b = d.bb.addButton(_('Restart calibre now'), d.bb.AcceptRole)
-        b.setIcon(QIcon(I('lt.png')))
-        d.do_restart = False
-        def rf():
-            d.do_restart = True
-        b.clicked.connect(rf)
-        d.set_details('')
-        d.exec_()
-        b.clicked.disconnect()
-        self.close()
-        if d.do_restart:
-            self.gui.quit(restart=True)
-        

calibre-plugin/config.py

@@ -1,918 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
-__docformat__ = 'restructuredtext en'
-
-import traceback, copy
-from collections import OrderedDict
-
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
-
-from calibre.gui2 import dynamic, info_dialog
-from calibre.utils.config import JSONConfig
-from calibre.gui2.ui import get_gui
-
-from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order)
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
-
-from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
-
-from calibre.gui2.complete import MultiCompleteLineEdit
-
-PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
-PREFS_KEY_SETTINGS = 'settings'
-
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-default_prefs = {}
-default_prefs['personal.ini'] = get_resources('plugin-example.ini')
-
-default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
-default_prefs['updateepubcover'] = False
-default_prefs['keeptags'] = False
-default_prefs['urlsfromclip'] = True
-default_prefs['updatedefault'] = True
-default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = OVERWRITE
-default_prefs['deleteotherforms'] = False
-default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
-default_prefs['lookforurlinhtml'] = False
-default_prefs['injectseries'] = False
-
-default_prefs['send_lists'] = ''
-default_prefs['read_lists'] = ''
-default_prefs['addtolists'] = False
-default_prefs['addtoreadlists'] = False
-default_prefs['addtolistsonread'] = False
-
-default_prefs['gcnewonly'] = False
-default_prefs['gc_site_settings'] = {}
-default_prefs['allow_gc_from_ini'] = True
-
-default_prefs['countpagesstats'] = []
-
-default_prefs['errorcol'] = ''
-default_prefs['custom_cols'] = {}
-default_prefs['custom_cols_newonly'] = {}
-default_prefs['allow_custcol_from_ini'] = True
-
-default_prefs['std_cols_newonly'] = {}
-
-def set_library_config(library_config):
-    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
-                                              PREFS_KEY_SETTINGS,
-                                              library_config)
-    
-def get_library_config():
-    db = get_gui().current_db
-    library_id = get_library_uuid(db)
-    library_config = None
-    # Check whether this is a configuration needing to be migrated
-    # from json into database.  If so: get it, set it, rename it in json.
-    if library_id in old_prefs:
-        #print("get prefs from old_prefs")
-        library_config = old_prefs[library_id]
-        set_library_config(library_config)
-        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
-        del old_prefs[library_id]
-
-    if library_config is None:
-        #print("get prefs from db")
-        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
-                                                 copy.deepcopy(default_prefs))
-    return library_config
-
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# fake out so I don't have to change the prefs calls anywhere.  The
-# Java programmer in me is offended by op-overloading, but it's very
-# tidy.
-class PrefsFacade():
-    def __init__(self,default_prefs):
-        self.default_prefs = default_prefs
-        self.libraryid = None
-        self.current_prefs = None
-        
-    def _get_prefs(self):
-        libraryid = get_library_uuid(get_gui().current_db)
-        if self.current_prefs == None or self.libraryid != libraryid:
-            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
-            self.libraryid = libraryid
-            self.current_prefs = get_library_config()
-        return self.current_prefs
-        
-    def __getitem__(self,k):            
-        prefs = self._get_prefs()
-        if k not in prefs:
-            # pulls from default_prefs.defaults automatically if not set
-            # in default_prefs
-            return self.default_prefs[k]
-        return prefs[k]
-
-    def __setitem__(self,k,v):
-        prefs = self._get_prefs()
-        prefs[k]=v
-        # self._save_prefs(prefs)
-
-    def __delitem__(self,k):
-        prefs = self._get_prefs()
-        if k in prefs:
-            del prefs[k]
-
-    def save_to_db(self):
-        set_library_config(self._get_prefs())
-        
-
-prefs = PrefsFacade(default_prefs)
-    
-class ConfigWidget(QWidget):
-
-    def __init__(self, plugin_action):
-        QWidget.__init__(self)
-        self.plugin_action = plugin_action
-        
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">List of Supported Sites</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>')
-        label.setOpenExternalLinks(True)
-        self.l.addWidget(label)
-
-        tab_widget = QTabWidget(self)
-        self.l.addWidget(tab_widget)
-
-        self.basic_tab = BasicTab(self, plugin_action)
-        tab_widget.addTab(self.basic_tab, 'Basic')
-
-        self.personalini_tab = PersonalIniTab(self, plugin_action)
-        tab_widget.addTab(self.personalini_tab, 'personal.ini')
-        
-        self.readinglist_tab = ReadingListTab(self, plugin_action)
-        tab_widget.addTab(self.readinglist_tab, 'Reading Lists')
-        if 'Reading List' not in plugin_action.gui.iactions:
-            self.readinglist_tab.setEnabled(False)
-
-        self.generatecover_tab = GenerateCoverTab(self, plugin_action)
-        tab_widget.addTab(self.generatecover_tab, 'Generate Cover')
-        if 'Generate Cover' not in plugin_action.gui.iactions:
-            self.generatecover_tab.setEnabled(False)
-
-        self.countpages_tab = CountPagesTab(self, plugin_action)
-        tab_widget.addTab(self.countpages_tab, 'Count Pages')
-        if 'Count Pages' not in plugin_action.gui.iactions:
-            self.countpages_tab.setEnabled(False)
-
-        self.std_columns_tab = StandardColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.std_columns_tab, 'Standard Columns')
-
-        self.cust_columns_tab = CustomColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.cust_columns_tab, 'Custom Columns')
-
-        self.other_tab = OtherTab(self, plugin_action)
-        tab_widget.addTab(self.other_tab, 'Other')
-
-
-    def save_settings(self):
-
-        # basic
-        prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
-        prefs['collision'] = unicode(self.basic_tab.collision.currentText())
-        prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
-        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
-        prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
-        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
-        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
-        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
-        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
-        prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
-        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
-        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
-        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
-
-        if self.readinglist_tab:
-            # lists
-            prefs['send_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.readinglist_tab.send_lists_box.text()).split(','))))
-            prefs['read_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.readinglist_tab.read_lists_box.text()).split(','))))
-            # print("send_lists: %s"%prefs['send_lists'])
-            # print("read_lists: %s"%prefs['read_lists'])
-            prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked()
-            prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked()
-            prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()
-
-        # personal.ini
-        ini = unicode(self.personalini_tab.ini.toPlainText())
-        if ini:
-            prefs['personal.ini'] = ini
-        else:
-            # if they've removed everything, reset to default.
-            prefs['personal.ini'] = get_resources('plugin-example.ini')
-
-        # Generate Covers tab
-        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
-        gc_site_settings = {}
-        for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
-            if val != 'none':
-                gc_site_settings[site] = val
-                #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
-        prefs['gc_site_settings'] = gc_site_settings
-        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
-
-        # Count Pages tab
-        countpagesstats = []
-        
-        if self.countpages_tab.pagecount.isChecked():
-            countpagesstats.append('PageCount')
-        if self.countpages_tab.wordcount.isChecked():
-            countpagesstats.append('WordCount')
-        if self.countpages_tab.fleschreading.isChecked():
-            countpagesstats.append('FleschReading')
-        if self.countpages_tab.fleschgrade.isChecked():
-            countpagesstats.append('FleschGrade')
-        if self.countpages_tab.gunningfog.isChecked():
-            countpagesstats.append('GunningFog')
-            
-        prefs['countpagesstats'] = countpagesstats
-        
-        # Standard Columns tab
-        colsnewonly = {}
-        for (col,checkbox) in self.std_columns_tab.stdcol_newonlycheck.iteritems():
-            colsnewonly[col] = checkbox.isChecked()
-        prefs['std_cols_newonly'] = colsnewonly
-
-        # Custom Columns tab
-        # error column
-        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
-
-        # cust cols tab
-        colsmap = {}
-        for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
-            if val != 'none':
-                colsmap[col] = val
-                #print("colsmap[%s]:%s"%(col,colsmap[col]))
-        prefs['custom_cols'] = colsmap
-
-        colsnewonly = {}
-        for (col,checkbox) in self.cust_columns_tab.custcol_newonlycheck.iteritems():
-            colsnewonly[col] = checkbox.isChecked()
-        prefs['custom_cols_newonly'] = colsnewonly
-        
-        prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
-        
-        prefs.save_to_db()
-
-    def edit_shortcuts(self):
-        self.save_settings()
-        # Force the menus to be rebuilt immediately, so we have all our actions registered
-        self.plugin_action.rebuild_menus()
-        d = KeyboardConfigDialog(self.plugin_action.gui, self.plugin_action.action_spec[0])
-        if d.exec_() == d.Accepted:
-            self.plugin_action.gui.keyboard.finalize()
-
-class BasicTab(QWidget):
-
-    def __init__(self, parent_dialog, plugin_action):
-        self.parent_dialog = parent_dialog
-        self.plugin_action = plugin_action
-        QWidget.__init__(self)
-        
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-
-        tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
-        horz = QHBoxLayout()
-        label = QLabel('Default Output &Format:')
-        label.setToolTip(tooltip)
-        horz.addWidget(label)
-        self.fileform = QComboBox(self)
-        self.fileform.addItem('epub')
-        self.fileform.addItem('mobi')
-        self.fileform.addItem('html')
-        self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip(tooltip)
-        self.fileform.activated.connect(self.set_collisions)
-        label.setBuddy(self.fileform)
-        horz.addWidget(self.fileform)
-        self.l.addLayout(horz)
-
-        tooltip = "On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to."
-        horz = QHBoxLayout()
-        label = QLabel('Default If Story Already Exists?')
-        label.setToolTip(tooltip)
-        horz.addWidget(label)
-        self.collision = QComboBox(self)
-        # add collision options
-        self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
-        if i > -1:
-            self.collision.setCurrentIndex(i)
-        self.collision.setToolTip(tooltip)
-        label.setBuddy(self.collision)
-        horz.addWidget(self.collision)
-        self.l.addLayout(horz)
-
-        self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books.")
-        self.updatemeta.setChecked(prefs['updatemeta'])
-        self.l.addWidget(self.updatemeta)
-
-        self.updateepubcover = QCheckBox('Default Update EPUB Cover when Updating EPUB?',self)
-        self.updateepubcover.setToolTip("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off.")
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
-        self.l.addWidget(self.updateepubcover)
-
-        self.l.addSpacing(10)        
-
-        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
-        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
-        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
-        self.l.addWidget(self.deleteotherforms)
-        
-        self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
-        self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
-        self.updatecover.setChecked(prefs['updatecover'])
-        self.l.addWidget(self.updatecover)
-
-        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
-        self.keeptags.setToolTip("Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")
-        self.keeptags.setChecked(prefs['keeptags'])
-        self.l.addWidget(self.keeptags)
-
-        self.l.addSpacing(10)        
-
-        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
-        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
-        self.urlsfromclip.setChecked(prefs['urlsfromclip'])
-        self.l.addWidget(self.urlsfromclip)
-
-        self.updatedefault = QCheckBox('Default to Update when books selected?',self)
-        self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+
-                                      'books are selected.  If unchecked, it will always bring up \'Add New\'.')
-        self.updatedefault.setChecked(prefs['updatedefault'])
-        self.l.addWidget(self.updatedefault)
-
-        self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
-        self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
-        self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
-        self.l.addWidget(self.adddialogstaysontop)
-
-        self.l.addSpacing(10)        
-
-        # this is a cheat to make it easier for users to realize there's a new include_images features.
-        self.includeimages = QCheckBox("Include images in EPUBs?",self)
-        self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
-        self.includeimages.setChecked(prefs['includeimages'])
-        self.l.addWidget(self.includeimages)
-
-        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
-        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
-        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
-        self.l.addWidget(self.lookforurlinhtml)
-
-        self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
-        self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
-        self.injectseries.setChecked(prefs['injectseries'])
-        self.l.addWidget(self.injectseries)
-
-        self.l.insertStretch(-1)
-        
-    def set_collisions(self):
-        prev=self.collision.currentText()
-        self.collision.clear()
-        for o in collision_order:
-            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
-                self.collision.addItem(o)
-        i = self.collision.findText(prev)
-        if i > -1:
-            self.collision.setCurrentIndex(i)
-        
-    def show_defaults(self):
-        text = get_resources('plugin-defaults.ini')
-        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-        
-class PersonalIniTab(QWidget):
-
-    def __init__(self, parent_dialog, plugin_action):
-        self.parent_dialog = parent_dialog
-        self.plugin_action = plugin_action
-        QWidget.__init__(self)
-        
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        label = QLabel('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set is_adult and user/password for different sites.')
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-        
-        self.label = QLabel('personal.ini:')
-        self.l.addWidget(self.label)
-
-        self.ini = QTextEdit(self)
-        try:
-            self.ini.setFont(QFont("Courier",
-                                   self.plugin_action.gui.font().pointSize()+1));
-        except Exception as e:
-            print("Couldn't get font: %s"%e)
-        self.ini.setLineWrapMode(QTextEdit.NoWrap)
-        self.ini.setText(prefs['personal.ini'])
-        self.l.addWidget(self.ini)
-
-        self.defaults = QPushButton('View Defaults (plugin-defaults.ini)', self)
-        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
-        self.defaults.clicked.connect(self.show_defaults)
-        self.l.addWidget(self.defaults)
-        
-        # self.l.insertStretch(-1)
-        # let edit box fill the space.
-        
-    def show_defaults(self):
-        text = get_resources('plugin-defaults.ini')
-        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-
-class ShowDefaultsIniDialog(QDialog):
-
-    def __init__(self, icon, text, parent=None):
-        QDialog.__init__(self, parent)
-        self.resize(600, 500)
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-        self.label = QLabel("Plugin Defaults (plugin-defaults.ini) (Read-Only)")
-        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
-        self.setWindowTitle(_('Plugin Defaults'))
-        self.setWindowIcon(icon)
-        self.l.addWidget(self.label)
-        
-        self.ini = QTextEdit(self)
-        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
-        try:
-            self.ini.setFont(QFont("Courier",
-                                   get_gui().font().pointSize()+1));
-        except Exception as e:
-            print("Couldn't get font: %s"%e)
-        self.ini.setLineWrapMode(QTextEdit.NoWrap)
-        self.ini.setText(text)
-        self.ini.setReadOnly(True)
-        self.l.addWidget(self.ini)
-        
-        self.ok_button = QPushButton('OK', self)
-        self.ok_button.clicked.connect(self.hide)
-        self.l.addWidget(self.ok_button)
-        
-class ReadingListTab(QWidget):
-
-    def __init__(self, parent_dialog, plugin_action):
-        self.parent_dialog = parent_dialog
-        self.plugin_action = plugin_action
-        QWidget.__init__(self)
-        
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        try:
-            rl_plugin = plugin_action.gui.iactions['Reading List']
-            reading_lists = rl_plugin.get_list_names()
-        except KeyError:
-            reading_lists= []
-            
-        label = QLabel('These settings provide integration with the Reading List Plugin.  Reading List can automatically send to devices and change custom columns.  You have to create and configure the lists in Reading List to be useful.')
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-        
-        self.addtolists = QCheckBox('Add new/updated stories to "Send to Device" Reading List(s).',self)
-        self.addtolists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.')
-        self.addtolists.setChecked(prefs['addtolists'])
-        self.l.addWidget(self.addtolists)
-            
-        horz = QHBoxLayout()
-        label = QLabel('"Send to Device" Reading Lists')
-        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
-        horz.addWidget(label)        
-        self.send_lists_box = MultiCompleteLineEdit(self)
-        self.send_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
-        self.send_lists_box.update_items_cache(reading_lists)
-        self.send_lists_box.setText(prefs['send_lists'])
-        horz.addWidget(self.send_lists_box)
-        self.l.addLayout(horz)
-        
-        self.addtoreadlists = QCheckBox('Add new/updated stories to "To Read" Reading List(s).',self)
-        self.addtoreadlists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')
-        self.addtoreadlists.setChecked(prefs['addtoreadlists'])
-        self.l.addWidget(self.addtoreadlists)
-            
-        horz = QHBoxLayout()
-        label = QLabel('"To Read" Reading Lists')
-        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
-        horz.addWidget(label)        
-        self.read_lists_box = MultiCompleteLineEdit(self)
-        self.read_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
-        self.read_lists_box.update_items_cache(reading_lists)
-        self.read_lists_box.setText(prefs['read_lists'])
-        horz.addWidget(self.read_lists_box)
-        self.l.addLayout(horz)
-        
-        self.addtolistsonread = QCheckBox('Add stories back to "Send to Device" Reading List(s) when marked "Read".',self)
-        self.addtolistsonread.setToolTip('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)')
-        self.addtolistsonread.setChecked(prefs['addtolistsonread'])
-        self.l.addWidget(self.addtolistsonread)
-            
-        self.l.insertStretch(-1)
-        
-class GenerateCoverTab(QWidget):
-
-    def __init__(self, parent_dialog, plugin_action):
-        self.parent_dialog = parent_dialog
-        self.plugin_action = plugin_action
-        QWidget.__init__(self)
-        
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        try:
-            gc_plugin = plugin_action.gui.iactions['Generate Cover']
-            gc_settings = gc_plugin.get_saved_setting_names()
-        except KeyError:
-            gc_settings= []
-            
-        label = QLabel('The Generate Cover plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-
-        scrollable = QScrollArea()
-        scrollcontent = QWidget()
-        scrollable.setWidget(scrollcontent)
-        scrollable.setWidgetResizable(True)
-        self.l.addWidget(scrollable)
-
-        self.sl = QVBoxLayout()
-        scrollcontent.setLayout(self.sl)
-        
-        self.gc_dropdowns = {}
-
-        sitelist = getConfigSections()
-        sitelist.sort()
-        sitelist.insert(0,u"Default")
-        for site in sitelist:
-            horz = QHBoxLayout()
-            label = QLabel(site)
-            if site == u"Default":
-                s = "On Metadata update, run Generate Cover with this setting, if not selected for specific site."
-            else:
-                s = "On Metadata update, run Generate Cover with this setting for %s stories."%site
-
-            label.setToolTip(s)
-            horz.addWidget(label)
-            dropdown = QComboBox(self)
-            dropdown.setToolTip(s)
-            dropdown.addItem('',QVariant('none'))
-            for setting in gc_settings:
-                dropdown.addItem(setting,QVariant(setting))
-            self.gc_dropdowns[site] = dropdown
-            if site in prefs['gc_site_settings']:
-                dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
-
-            horz.addWidget(dropdown)
-            self.sl.addLayout(horz)
-        
-        self.sl.insertStretch(-1)
-        
-        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
-        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
-        self.gcnewonly.setChecked(prefs['gcnewonly'])
-        self.l.addWidget(self.gcnewonly)
-
-        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override',self)
-        self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
-        self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
-        self.l.addWidget(self.allow_gc_from_ini)
-            
-class CountPagesTab(QWidget):
-
-    def __init__(self, parent_dialog, plugin_action):
-        self.parent_dialog = parent_dialog
-        self.plugin_action = plugin_action
-        QWidget.__init__(self)
-        
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        label = QLabel('These settings provide integration with the Count Pages Plugin.  Count Pages can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in Count Pages first.')
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-
-        label = QLabel('If any of the settings below are checked, when stories are added or updated, the Count Pages Plugin will be called to update the checked statistics.')
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-        
-        # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
-        self.pagecount = QCheckBox('Page Count',self)
-        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
-        self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
-        self.l.addWidget(self.pagecount)
-            
-        self.wordcount = QCheckBox('Word Count',self)
-        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
-        self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
-        self.l.addWidget(self.wordcount)
-
-        self.fleschreading = QCheckBox('Flesch Reading Ease',self)
-        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
-        self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
-        self.l.addWidget(self.fleschreading)
-        
-        self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
-        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
-        self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
-        self.l.addWidget(self.fleschgrade)
-        
-        self.gunningfog = QCheckBox('Gunning Fog Index',self)
-        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
-        self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
-        self.l.addWidget(self.gunningfog)
-        
-        self.l.insertStretch(-1)
-        
-class OtherTab(QWidget):
-
-    def __init__(self, parent_dialog, plugin_action):
-        self.parent_dialog = parent_dialog
-        self.plugin_action = plugin_action
-        QWidget.__init__(self)
-        
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        label = QLabel("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again.")
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-        
-        keyboard_shortcuts_button = QPushButton('Keyboard shortcuts...', self)
-        keyboard_shortcuts_button.setToolTip(_(
-                    'Edit the keyboard shortcuts associated with this plugin'))
-        keyboard_shortcuts_button.clicked.connect(parent_dialog.edit_shortcuts)
-        self.l.addWidget(keyboard_shortcuts_button)
-
-        reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
-        reset_confirmation_button.setToolTip(_(
-                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
-        reset_confirmation_button.clicked.connect(self.reset_dialogs)
-        self.l.addWidget(reset_confirmation_button)
-        
-        view_prefs_button = QPushButton('&View library preferences...', self)
-        view_prefs_button.setToolTip(_(
-                    'View data stored in the library database for this plugin'))
-        view_prefs_button.clicked.connect(self.view_prefs)
-        self.l.addWidget(view_prefs_button)
-        
-        self.l.insertStretch(-1)
-        
-    def reset_dialogs(self):
-        for key in dynamic.keys():
-            if key.startswith('fanfictiondownloader_') and key.endswith('_again') \
-                                                  and dynamic[key] is False:
-                dynamic[key] = True
-        info_dialog(self, _('Done'),
-                    _('Confirmation dialogs have all been reset'),
-                    show=True,
-                    show_copy_button=False)
-        
-    def view_prefs(self):
-        d = PrefsViewerDialog(self.plugin_action.gui, PREFS_NAMESPACE)
-        d.exec_()
-
-permitted_values = {
-    'int' : ['numWords','numChapters'],
-    'float' : ['numWords','numChapters'],
-    'bool' : ['status-C','status-I'],
-    'datetime' : ['datePublished', 'dateUpdated', 'dateCreated'],
-    'series' : ['series'],
-    'enumeration' : ['category',
-                     'genre',
-                     'language',
-                     'series',
-                     'characters',
-                     'ships',
-                     'status',
-                     'datePublished',
-                     'dateUpdated',
-                     'dateCreated',
-                     'rating',
-                     'warnings',
-                     'numChapters',
-                     'numWords',
-                     'site',
-                     'storyId',
-                     'authorId',
-                     'extratags',
-                     'title',
-                     'storyUrl',
-                     'description',
-                     'author',
-                     'authorUrl',
-                     'formatname',
-                     'version'
-                     #,'formatext'   # not useful information.
-                     #,'siteabbrev'
-                     ]
-    }
-# no point copying the whole list.
-permitted_values['text'] = permitted_values['enumeration']
-permitted_values['comments'] = permitted_values['enumeration']
-
-titleLabels = {
-    'category':'Category',
-    'genre':'Genre',
-    'language':'Language',
-    'status':'Status',
-    'status-C':'Status:Completed',
-    'status-I':'Status:In-Progress',
-    'series':'Series',
-    'characters':'Characters',
-    'ships':'Relationships',
-    'datePublished':'Published',
-    'dateUpdated':'Updated',
-    'dateCreated':'Packaged',
-    'rating':'Rating',
-    'warnings':'Warnings',
-    'numChapters':'Chapters',
-    'numWords':'Words',
-    'site':'Site',
-    'storyId':'Story ID',
-    'authorId':'Author ID',
-    'extratags':'Extra Tags',
-    'title':'Title',
-    'storyUrl':'Story URL',
-    'description':'Summary',
-    'author':'Author',
-    'authorUrl':'Author URL',
-    'formatname':'File Format',
-    'formatext':'File Extension',
-    'siteabbrev':'Site Abbrev',
-    'version':'FFDL Version'
-    }
-
-class CustomColumnsTab(QWidget):
-
-    def __init__(self, parent_dialog, plugin_action):
-        self.parent_dialog = parent_dialog
-        self.plugin_action = plugin_action
-        QWidget.__init__(self)
-        
-        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
-
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        label = QLabel("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically.")
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-        
-        self.custcol_dropdowns = {}
-        self.custcol_newonlycheck = {}
-
-        scrollable = QScrollArea()
-        scrollcontent = QWidget()
-        scrollable.setWidget(scrollcontent)
-        scrollable.setWidgetResizable(True)
-        self.l.addWidget(scrollable)
-
-        self.sl = QVBoxLayout()
-        scrollcontent.setLayout(self.sl)
-        
-        for key, column in custom_columns.iteritems():
-
-            if column['datatype'] in permitted_values:
-                # print("\n============== %s ===========\n"%key)
-                # for (k,v) in column.iteritems():
-                #     print("column['%s'] => %s"%(k,v))
-                horz = QHBoxLayout()
-                label = QLabel(column['name'])
-                label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
-                horz.addWidget(label)
-                dropdown = QComboBox(self)
-                dropdown.addItem('',QVariant('none'))
-                for md in permitted_values[column['datatype']]:
-                    dropdown.addItem(titleLabels[md],QVariant(md))
-                self.custcol_dropdowns[key] = dropdown
-                if key in prefs['custom_cols']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
-                if column['datatype'] == 'enumeration':
-                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
-                else:
-                    dropdown.setToolTip("Metadata values valid for this type of column.")
-                horz.addWidget(dropdown)
-
-                newonlycheck = QCheckBox("New Only",self)
-                newonlycheck.setToolTip("Write to %s(%s) only for new\nbooks, not updates to existing books."%(column['name'],key))
-                self.custcol_newonlycheck[key] = newonlycheck
-                if key in prefs['custom_cols_newonly']:
-                    newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
-                horz.addWidget(newonlycheck)
-                
-                self.sl.addLayout(horz)
-        
-        self.sl.insertStretch(-1)
-
-        self.l.addSpacing(5)
-        self.allow_custcol_from_ini = QCheckBox('Allow custom_columns_settings from personal.ini to override',self)
-        self.allow_custcol_from_ini.setToolTip("The personal.ini parameter custom_columns_settings allows you to set custom columns to site specific values that aren't common to all sites.<br \>custom_columns_settings is ignored when this is off.")
-        self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
-        self.l.addWidget(self.allow_custcol_from_ini)
-        
-        self.l.addSpacing(5)
-        label = QLabel("Special column:")
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        
-        horz = QHBoxLayout()
-        label = QLabel("Update/Overwrite Error Column:")
-        tooltip="When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)"
-        label.setToolTip(tooltip)
-        horz.addWidget(label)
-        self.errorcol = QComboBox(self)
-        self.errorcol.setToolTip(tooltip)
-        self.errorcol.addItem('',QVariant('none'))
-        for key, column in custom_columns.iteritems():
-            if column['datatype'] in ('text','comments'):
-                self.errorcol.addItem(column['name'],QVariant(key))
-        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
-        horz.addWidget(self.errorcol)
-        self.l.addLayout(horz)
-
-        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
-
-
-class StandardColumnsTab(QWidget):
-
-    def __init__(self, parent_dialog, plugin_action):
-        self.parent_dialog = parent_dialog
-        self.plugin_action = plugin_action
-        QWidget.__init__(self)
-
-        columns=OrderedDict()
-        
-        columns["title"]="Title"
-        columns["authors"]="Author(s)"
-        columns["publisher"]="Publisher"
-        columns["tags"]="Tags"
-        columns["languages"]="Languages"
-        columns["pubdate"]="Published Date"
-        columns["timestamp"]="Date"
-        columns["comments"]="Comments"
-        columns["series"]="Series"
-        columns["identifiers"]="Ids(url id only)"
-
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        label = QLabel("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books.")
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
-        
-        self.stdcol_newonlycheck = {}
-
-        for key, column in columns.iteritems():
-            horz = QHBoxLayout()
-            label = QLabel(column)
-            #label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
-            horz.addWidget(label)
-
-            newonlycheck = QCheckBox("New Only",self)
-            newonlycheck.setToolTip("Write to %s only for new\nbooks, not updates to existing books."%column)
-            self.stdcol_newonlycheck[key] = newonlycheck
-            if key in prefs['std_cols_newonly']:
-                newonlycheck.setChecked(prefs['std_cols_newonly'][key])
-            horz.addWidget(newonlycheck)
-            
-            self.l.addLayout(horz)
-        
-        self.l.insertStretch(-1)

calibre-plugin/dialogs.py

@@ -1,714 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
-__docformat__ = 'restructuredtext en'
-
-import traceback
-
-from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
-                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView )
-
-from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
-from calibre.gui2.dialogs.confirm_delete import confirm
-
-from calibre import confirm_config_name
-from calibre.gui2 import dynamic
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
-from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
-            ImageTitleLayout, get_icon)
-
-SKIP='Skip'
-ADDNEW='Add New Book'
-UPDATE='Update EPUB if New Chapters'
-UPDATEALWAYS='Update EPUB Always'
-OVERWRITE='Overwrite if Newer'
-OVERWRITEALWAYS='Overwrite Always'
-CALIBREONLY='Update Calibre Metadata Only'
-collision_order=[SKIP,
-                 ADDNEW,
-                 UPDATE,
-                 UPDATEALWAYS,
-                 OVERWRITE,
-                 OVERWRITEALWAYS,
-                 CALIBREONLY,]
-        
-class NotGoingToDownload(Exception):
-    def __init__(self,error,icon='dialog_error.png'):
-        self.error=error
-        self.icon=icon
-
-    def __str__(self):
-        return self.error
-
-class DroppableQTextEdit(QTextEdit):
-    def __init__(self,parent):
-        QTextEdit.__init__(self,parent)
-        
-    def canInsertFromMimeData(self, source):
-        if source.hasUrls():
-            return True;
-        else:
-            return QTextEdit.canInsertFromMimeData(self,source)
-
-    def insertFromMimeData(self, source):
-        if source.hasText():
-            self.append(source.text())
-        else:
-            return QTextEdit.insertFromMimeData(self, source)
-                            
-class AddNewDialog(SizePersistedDialog):
-
-    def __init__(self, gui, prefs, icon, url_list_text):
-        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
-        self.gui = gui
-
-        if prefs['adddialogstaysontop']:
-            QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
-        
-        self.setMinimumWidth(300)
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-
-        self.setWindowTitle('FanFictionDownLoader')
-        self.setWindowIcon(icon)
-
-        self.l.addWidget(QLabel('Story URL(s), one per line:'))
-        self.url = DroppableQTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
-        self.url.setLineWrapMode(QTextEdit.NoWrap)
-        self.url.setText(url_list_text)
-        self.l.addWidget(self.url)
-        
-        horz = QHBoxLayout()
-        label = QLabel('Output &Format:')
-        horz.addWidget(label)
-        self.fileform = QComboBox(self)
-        self.fileform.addItem('epub')
-        self.fileform.addItem('mobi')
-        self.fileform.addItem('html')
-        self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
-        self.fileform.activated.connect(self.set_collisions)
-        
-        label.setBuddy(self.fileform)
-        horz.addWidget(self.fileform)
-        self.l.addLayout(horz)
-
-        horz = QHBoxLayout()
-        label = QLabel('If Story Already Exists?')
-        horz.addWidget(label)
-        self.collision = QComboBox(self)
-        self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
-        # add collision options
-        self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
-        if i > -1:
-            self.collision.setCurrentIndex(i)
-        label.setBuddy(self.collision)
-        horz.addWidget(self.collision)
-        self.l.addLayout(horz)
-
-        horz = QHBoxLayout()
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
-        self.updatemeta.setChecked(prefs['updatemeta'])
-        horz.addWidget(self.updatemeta)
-
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
-        horz.addWidget(self.updateepubcover)
-        
-        self.l.addLayout(horz)
-
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-        self.l.addWidget(button_box)
-        
-        if url_list_text:
-            button_box.button(QDialogButtonBox.Ok).setFocus()
-
-        # restore saved size.
-        self.resize_dialog()
-        #self.resize(self.sizeHint())
-
-    def set_collisions(self):
-        prev=self.collision.currentText()
-        self.collision.clear()
-        for o in collision_order:
-            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
-                self.collision.addItem(o)
-        i = self.collision.findText(prev)
-        if i > -1:
-            self.collision.setCurrentIndex(i)
-        
-    def get_ffdl_options(self):
-        return {
-            'fileform': unicode(self.fileform.currentText()),
-            'collision': unicode(self.collision.currentText()),
-            'updatemeta': self.updatemeta.isChecked(),
-            'updateepubcover': self.updateepubcover.isChecked(),
-            }
-
-    def get_urlstext(self):
-        return unicode(self.url.toPlainText())
-
-
-class FakeLineEdit():
-    def __init__(self):
-        pass
-    
-    def text(self):
-        pass
-    
-class CollectURLDialog(SizePersistedDialog):
-    '''
-    Collect single url for get urls.
-    '''
-    def __init__(self, gui, title, url_text): 
-        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
-        self.gui = gui
-        self.status=False
-
-        self.setMinimumWidth(300)
-        
-        self.l = QGridLayout()
-        self.setLayout(self.l)
-
-        self.setWindowTitle(title)
-        self.l.addWidget(QLabel(title),0,0,1,2)
-        
-        self.l.addWidget(QLabel("URL:"),1,0)
-        self.url = QLineEdit(self)
-        self.url.setText(url_text)
-        self.l.addWidget(self.url,1,1)
-   
-        self.ok_button = QPushButton('OK', self)
-        self.ok_button.clicked.connect(self.ok)
-        self.l.addWidget(self.ok_button,2,0)
-
-        self.cancel_button = QPushButton('Cancel', self)
-        self.cancel_button.clicked.connect(self.cancel)
-        self.l.addWidget(self.cancel_button,2,1)
-
-        # restore saved size.
-        self.resize_dialog()
-
-    def ok(self):
-        self.status=True
-        self.accept()
-
-    def cancel(self):
-        self.status=False
-        self.reject()
-
-class UserPassDialog(QDialog):
-    '''
-    Need to collect User/Pass for some sites.
-    '''
-    def __init__(self, gui, site, exception=None):
-        QDialog.__init__(self, gui)
-        self.gui = gui
-        self.status=False
-
-        self.l = QGridLayout()
-        self.setLayout(self.l)
-
-        if exception.passwdonly:
-            self.setWindowTitle('Password')
-            self.l.addWidget(QLabel("Author requires a password for this story(%s)."%exception.url),0,0,1,2)
-            # user isn't used, but it's easier to still have it for
-            # post processing.
-            self.user = FakeLineEdit()
-        else:
-            self.setWindowTitle('User/Password')
-            self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
-        
-            self.l.addWidget(QLabel("User:"),1,0)
-            self.user = QLineEdit(self)
-            self.l.addWidget(self.user,1,1)
-   
-        self.l.addWidget(QLabel("Password:"),2,0)
-        self.passwd = QLineEdit(self)
-        self.passwd.setEchoMode(QLineEdit.Password)
-        self.l.addWidget(self.passwd,2,1)
-   
-        self.ok_button = QPushButton('OK', self)
-        self.ok_button.clicked.connect(self.ok)
-        self.l.addWidget(self.ok_button,3,0)
-
-        self.cancel_button = QPushButton('Cancel', self)
-        self.cancel_button.clicked.connect(self.cancel)
-        self.l.addWidget(self.cancel_button,3,1)
-
-        self.resize(self.sizeHint())
-
-    def ok(self):
-        self.status=True
-        self.hide()
-
-    def cancel(self):
-        self.status=False
-        self.hide()
-
-class LoopProgressDialog(QProgressDialog):
-    '''
-    ProgressDialog displayed while fetching metadata for each story.
-    '''
-    def __init__(self, gui,
-                 book_list,
-                 foreach_function,
-                 finish_function,
-                 init_label="Fetching metadata for stories...",
-                 win_title="Downloading metadata for stories",
-                 status_prefix="Fetched metadata for"):
-        QProgressDialog.__init__(self,
-                                 init_label,
-                                 QString(), 0, len(book_list), gui)
-        self.setWindowTitle(win_title)
-        self.setMinimumWidth(500)
-        self.gui = gui
-        self.book_list = book_list
-        self.foreach_function = foreach_function
-        self.finish_function = finish_function
-        self.status_prefix = status_prefix
-        self.i = 0
-        
-        ## self.do_loop does QTimer.singleShot on self.do_loop also.
-        ## A weird way to do a loop, but that was the example I had.
-        QTimer.singleShot(0, self.do_loop)
-        self.exec_()
-
-    def updateStatus(self):
-        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
-        self.setValue(self.i+1)
-        print(self.labelText())
-
-    def do_loop(self):
-
-        if self.i == 0:
-            self.setValue(0)
-
-        book = self.book_list[self.i]
-        try:
-            ## collision spec passed into getadapter by partial from ffdl_plugin
-            ## no retval only if it exists, but collision is SKIP
-            self.foreach_function(book)
-            
-        except NotGoingToDownload as d:
-            book['good']=False
-            book['comment']=unicode(d)
-            book['icon'] = d.icon
-
-        except Exception as e:
-            book['good']=False
-            book['comment']=unicode(e)
-            print("Exception: %s:%s"%(book,unicode(e)))
-            traceback.print_exc()
-            
-        self.updateStatus()
-        self.i += 1
-            
-        if self.i >= len(self.book_list) or self.wasCanceled():
-            return self.do_when_finished()
-        else:
-            QTimer.singleShot(0, self.do_loop)
-
-    def do_when_finished(self):
-        self.hide()
-        self.gui = None        
-        # Queues a job to process these books in the background.
-        self.finish_function(self.book_list)
-
-class AboutDialog(QDialog):
-
-    def __init__(self, parent, icon, text):
-        QDialog.__init__(self, parent)
-        self.resize(400, 250)
-        self.l = QGridLayout()
-        self.setLayout(self.l)
-        self.logo = QLabel()
-        self.logo.setMaximumWidth(110)
-        self.logo.setPixmap(QPixmap(icon.pixmap(100,100)))
-        self.label = QLabel(text)
-        self.label.setOpenExternalLinks(True)
-        self.label.setWordWrap(True)
-        self.setWindowTitle(_('About FanFictionDownLoader'))
-        self.setWindowIcon(icon)
-        self.l.addWidget(self.logo, 0, 0)
-        self.l.addWidget(self.label, 0, 1)
-        self.bb = QDialogButtonBox(self)
-        b = self.bb.addButton(_('OK'), self.bb.AcceptRole)
-        b.setDefault(True)
-        self.l.addWidget(self.bb, 2, 0, 1, -1)
-        self.bb.accepted.connect(self.accept)
-
-class IconWidgetItem(ReadOnlyTextIconWidgetItem):
-    def __init__(self, text, icon, sort_key):
-        ReadOnlyTextIconWidgetItem.__init__(self, text, icon)
-        self.sort_key = sort_key
-
-    #Qt uses a simple < check for sorting items, override this to use the sortKey
-    def __lt__(self, other):
-        return self.sort_key < other.sort_key
-
-class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
-    def __init__(self, text, sort_key):
-        ReadOnlyTableWidgetItem.__init__(self, text)
-        self.sort_key = sort_key
-
-    #Qt uses a simple < check for sorting items, override this to use the sortKey
-    def __lt__(self, other):
-        return self.sort_key.lower() < other.sort_key.lower()
-
-class UpdateExistingDialog(SizePersistedDialog):
-    def __init__(self, gui, header, prefs, icon, books,
-                 save_size_name='fanfictiondownloader_plugin:update list dialog'):
-        SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.gui = gui
-        
-        self.setWindowTitle(header)
-        self.setWindowIcon(icon)
-        
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-        title_layout = ImageTitleLayout(self, 'images/icon.png',
-                                        header)
-        layout.addLayout(title_layout)
-        books_layout = QHBoxLayout()
-        layout.addLayout(books_layout)
-
-        self.books_table = StoryListTableWidget(self)
-        books_layout.addWidget(self.books_table)
-
-        button_layout = QVBoxLayout()
-        books_layout.addLayout(button_layout)
-        # self.move_up_button = QtGui.QToolButton(self)
-        # self.move_up_button.setToolTip('Move selected books up the list')
-        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
-        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
-        # button_layout.addWidget(self.move_up_button)
-        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
-        button_layout.addItem(spacerItem)
-        self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected books from the list')
-        self.remove_button.setIcon(get_icon('list_remove.png'))
-        self.remove_button.clicked.connect(self.remove_from_list)
-        button_layout.addWidget(self.remove_button)
-        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
-        button_layout.addItem(spacerItem1)
-        # self.move_down_button = QtGui.QToolButton(self)
-        # self.move_down_button.setToolTip('Move selected books down the list')
-        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
-        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
-        # button_layout.addWidget(self.move_down_button)
-
-        options_layout = QHBoxLayout()
-
-        label = QLabel('Output &Format:')
-        options_layout.addWidget(label)
-        self.fileform = QComboBox(self)
-        self.fileform.addItem('epub')
-        self.fileform.addItem('mobi')
-        self.fileform.addItem('html')
-        self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
-        self.fileform.activated.connect(self.set_collisions)
-        label.setBuddy(self.fileform)
-        options_layout.addWidget(self.fileform)
-        
-        label = QLabel('Update Mode:')
-        options_layout.addWidget(label)
-        self.collision = QComboBox(self)
-        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
-        # add collision options
-        self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
-        if i > -1:
-            self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
-        label.setBuddy(self.collision)
-        options_layout.addWidget(self.collision)
-
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
-        self.updatemeta.setChecked(prefs['updatemeta'])
-        options_layout.addWidget(self.updatemeta)
-                
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
-        options_layout.addWidget(self.updateepubcover)
-        
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-        options_layout.addWidget(button_box)
-        
-        layout.addLayout(options_layout)
-        
-        # Cause our dialog size to be restored from prefs or created on first usage
-        self.resize_dialog()
-        self.books_table.populate_table(books)
-
-    def set_collisions(self):
-        prev=self.collision.currentText()
-        self.collision.clear()
-        for o in collision_order:
-            if o not in [ADDNEW,SKIP] and \
-                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
-                self.collision.addItem(o)        
-        i = self.collision.findText(prev)
-        if i > -1:
-            self.collision.setCurrentIndex(i)
-        
-    def remove_from_list(self):
-        self.books_table.remove_selected_rows()
-
-    def get_books(self):
-        return self.books_table.get_books()
-
-    def get_ffdl_options(self):
-        return {
-            'fileform': unicode(self.fileform.currentText()),
-            'collision': unicode(self.collision.currentText()),
-            'updatemeta': self.updatemeta.isChecked(),
-            'updateepubcover': self.updateepubcover.isChecked(),
-            }
-
-def display_story_list(gui, header, prefs, icon, books,
-                       label_text='',
-                       save_size_name='fanfictiondownloader_plugin:display list dialog',
-                       offer_skip=False):
-    all_good = True
-    for b in books:
-        if not b['good']:
-            all_good=False
-            break
-        
-    ## 
-    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
-        return True
-        pass
-            ## fake accept?
-    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
-                               label_text,
-                               save_size_name,
-                               offer_skip and all_good)
-    d.exec_()
-    return d.result() == d.Accepted
-
-class DisplayStoryListDialog(SizePersistedDialog):
-    def __init__(self, gui, header, prefs, icon, books,
-                 label_text='',
-                 save_size_name='fanfictiondownloader_plugin:display list dialog',
-                 offer_skip=False):
-        SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.name = save_size_name
-        self.gui = gui
-        
-        self.setWindowTitle(header)
-        self.setWindowIcon(icon)
-        
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-        title_layout = ImageTitleLayout(self, 'images/icon.png',
-                                        header)
-        layout.addLayout(title_layout)
-
-        self.books_table = StoryListTableWidget(self)
-        layout.addWidget(self.books_table)
-
-        options_layout = QHBoxLayout()
-        self.label = QLabel(label_text)
-        #self.label.setOpenExternalLinks(True)
-        #self.label.setWordWrap(True)
-        options_layout.addWidget(self.label)
-        
-        if offer_skip:
-            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
-            options_layout.addItem(spacerItem1)
-            self.again = QCheckBox('Show this again?',self)
-            self.again.setChecked(True)
-            self.again.stateChanged.connect(self.toggle)
-            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
-            options_layout.addWidget(self.again)
-        
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-
-        options_layout.addWidget(button_box)
-        
-        layout.addLayout(options_layout)
-        
-        # Cause our dialog size to be restored from prefs or created on first usage
-        self.resize_dialog()
-        self.books_table.populate_table(books)
-
-    def get_books(self):
-        return self.books_table.get_books()
-
-    def toggle(self, *args):
-        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
-
-    
-
-class StoryListTableWidget(QTableWidget):
-
-    def __init__(self, parent):
-        QTableWidget.__init__(self, parent)
-        self.setSelectionBehavior(QAbstractItemView.SelectRows)
-
-    def populate_table(self, books):
-        self.clear()
-        self.setAlternatingRowColors(True)
-        self.setRowCount(len(books))
-        header_labels = ['','Title', 'Author', 'URL', 'Comment']
-        self.setColumnCount(len(header_labels))
-        self.setHorizontalHeaderLabels(header_labels)
-        self.horizontalHeader().setStretchLastSection(True)
-        #self.verticalHeader().setDefaultSectionSize(24)
-        self.verticalHeader().hide()
-
-        self.books={}
-        for row, book in enumerate(books):
-            self.populate_table_row(row, book)
-            self.books[row] = book
-
-        # turning True breaks up/down.  Do we need either sorting or up/down?
-        self.setSortingEnabled(True)
-        self.resizeColumnsToContents()
-        self.setMinimumColumnWidth(1, 100)
-        self.setMinimumColumnWidth(2, 100)
-        self.setMinimumColumnWidth(3, 100)
-        self.setMinimumSize(300, 0)
-        # if len(books) > 0:
-        #     self.selectRow(0)
-        self.sortItems(1)
-        self.sortItems(0)
-
-    def setMinimumColumnWidth(self, col, minimum):
-        if self.columnWidth(col) < minimum:
-            self.setColumnWidth(col, minimum)
-
-    def populate_table_row(self, row, book):
-        if book['good']:
-            icon = get_icon('ok.png')
-            val = 0
-        else:
-            icon = get_icon('minus.png')
-            val = 1
-        if 'icon' in book:
-            icon = get_icon(book['icon'])
-
-        status_cell = IconWidgetItem(None,icon,val)
-        status_cell.setData(Qt.UserRole, QVariant(val))
-        self.setItem(row, 0, status_cell)
-        
-        title_cell = ReadOnlyTableWidgetItem(book['title'])
-        title_cell.setData(Qt.UserRole, QVariant(row))
-        self.setItem(row, 1, title_cell)
-        
-        self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
-        
-        url_cell = ReadOnlyTableWidgetItem(book['url'])
-        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
-        self.setItem(row, 3, url_cell)
-        
-        comment_cell = ReadOnlyTableWidgetItem(book['comment'])
-        #comment_cell.setData(Qt.UserRole, QVariant(book))
-        self.setItem(row, 4, comment_cell)
-
-    def get_books(self):
-        books = []
-        #print("=========================\nbooks:%s"%self.books)
-        for row in range(self.rowCount()):
-            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
-            book = self.books[rnum]
-            books.append(book)
-        return books
-
-    def remove_selected_rows(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        message = '<p>Are you sure you want to remove this book from the list?'
-        if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
-        if not confirm(message,'fanfictiondownloader_delete_item', self):
-            return
-        first_sel_row = self.currentRow()
-        for selrow in reversed(rows):
-            self.removeRow(selrow.row())
-        if first_sel_row < self.rowCount():
-            self.select_and_scroll_to_row(first_sel_row)
-        elif self.rowCount() > 0:
-            self.select_and_scroll_to_row(first_sel_row - 1)
-
-    def select_and_scroll_to_row(self, row):
-        self.selectRow(row)
-        self.scrollToItem(self.currentItem())
-
-    def move_rows_up(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        first_sel_row = rows[0].row()
-        if first_sel_row <= 0:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in selrows:
-            self.swap_row_widgets(selrow - 1, selrow + 1)
-        scroll_to_row = first_sel_row - 1
-        if scroll_to_row > 0:
-            scroll_to_row = scroll_to_row - 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def move_rows_down(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        last_sel_row = rows[-1].row()
-        if last_sel_row == self.rowCount() - 1:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in reversed(selrows):
-            self.swap_row_widgets(selrow + 2, selrow)
-        scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def swap_row_widgets(self, src_row, dest_row):
-        self.blockSignals(True)
-        self.insertRow(dest_row)
-        for col in range(0, self.columnCount()):
-            self.setItem(dest_row, col, self.takeItem(src_row, col))
-        self.removeRow(src_row)
-        self.blockSignals(False)

calibre-plugin/jobs.py

@@ -1,193 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
-__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
-__docformat__ = 'restructuredtext en'
-
-import time, os, traceback
-
-from StringIO import StringIO
-
-from calibre.utils.ipc.server import Server
-from calibre.utils.ipc.job import ParallelJob
-
-from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
-    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
-
-# ------------------------------------------------------------------------------
-#
-#              Functions to perform downloads using worker jobs
-#
-# ------------------------------------------------------------------------------
-
-def do_download_worker(book_list, options,
-                       cpus, notification=lambda x,y:x):
-    '''
-    Master job, to launch child jobs to extract ISBN for a set of books
-    This is run as a worker job in the background to keep the UI more
-    responsive and get around the memory leak issues as it will launch
-    a child job for each book as a worker process
-    '''
-    server = Server(pool_size=cpus)
-
-    print(options['version'])
-    total = 0
-    # Queue all the jobs
-    print("Adding jobs for URLs:")
-    for book in book_list:
-        if book['good']:
-            print("%s"%book['url'])
-            total += 1
-            args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
-                    'do_download_for_worker',
-                    (book,options)]
-            job = ParallelJob('arbitrary',
-                              "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
-                              done=None,
-                              args=args)
-            job._book = book
-            # job._book_id = book_id
-            # job._title = title
-            # job._modified_date = modified_date
-            # job._existing_isbn = existing_isbn
-            server.add_job(job)
-    
-    # This server is an arbitrary_n job, so there is a notifier available.
-    # Set the % complete to a small number to avoid the 'unavailable' indicator
-    notification(0.01, 'Downloading FanFiction Stories')
-
-    # dequeue the job results as they arrive, saving the results
-    count = 0
-    while True:
-        job = server.changed_jobs_queue.get()
-        # A job can 'change' when it is not finished, for example if it
-        # produces a notification. Ignore these.
-        job.update()
-        if not job.is_finished:
-            continue
-        # A job really finished. Get the information.
-        output_book = job.result
-        #print("output_book:%s"%output_book)
-        book_list.remove(job._book)
-        book_list.append(job.result)
-        book_id = job._book['calibre_id']
-        #title = job._title
-        count = count + 1
-        notification(float(count)/total, 'Downloaded Story')
-        # Add this job's output to the current log
-        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        print(job.details)
-
-        if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            print("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    print(book['title'])
-            print("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    print(book['title'])
-            break
-
-    server.close()
-    
-    # return the book list as the job result
-    return book_list
-
-def do_download_for_worker(book,options):
-    '''
-    Child job, to extract isbn from formats for this specific book,
-    when run as a worker job
-    '''
-    try:
-        book['comment'] = 'Download started...'
-
-        configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
-        configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        configuration.readfp(StringIO(options['personal.ini']))
-        
-        if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-            configuration.set("overrides","never_make_cover","true")
-
-        # images only for epub, even if the user mistakenly turned it
-        # on else where.
-        if options['fileform'] not in ("epub","html"):
-            configuration.set("overrides","include_images","false")
-        
-        adapter = adapters.getAdapter(configuration,book['url'])
-        adapter.is_adult = book['is_adult'] 
-        adapter.username = book['username'] 
-        adapter.password = book['password']
-        
-        story = adapter.getStoryMetadataOnly()
-        if 'calibre_series' in book:
-            adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
-            
-        # set PI version instead of default.
-        if 'version' in options:
-            story.setMetadata('version',options['version'])
-            
-        writer = writers.getWriter(options['fileform'],configuration,adapter)
-
-        outfile = book['outfile']
-
-        ## No need to download at all.  Shouldn't ever get down here.
-        if options['collision'] in (CALIBREONLY):
-            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
-            book['comment'] = 'Metadata collected.'
-            
-        ## checks were done earlier, it's new or not dup or newer--just write it.
-        elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
-                ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
-
-            adapter.setChaptersRange(book['begin'],book['end'])
-            
-            print("write to %s"%outfile)
-            writer.writeStory(outfilename=outfile, forceOverwrite=True)
-            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
-            
-        ## checks were done earlier, just update it.
-        elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-
-            # update now handled by pre-populating the old images and
-            # chapters in the adapter rather than merging epubs.
-            urlchaptercount = int(story.getMetadata('numChapters'))
-            (url,
-             chaptercount,
-             adapter.oldchapters,
-             adapter.oldimgs,
-             adapter.oldcover,
-             adapter.calibrebookmark,
-             adapter.logfile) = get_update_data(book['epub_for_update'])
-
-            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            print("write to %s"%outfile)
-
-            writer.writeStory(outfilename=outfile, forceOverwrite=True)
-            
-            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
-                (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
-        
-    except NotGoingToDownload as d:
-        book['good']=False
-        book['comment']=unicode(d)
-        book['icon'] = d.icon
-
-    except Exception as e:
-        book['good']=False
-        book['comment']=unicode(e)
-        book['icon']='dialog_error.png'
-        book['status'] = 'Error'
-        print("Exception: %s:%s"%(book,unicode(e)))
-        traceback.print_exc()
-
-    #time.sleep(10)
-    return book

constants.py

@@ -0,0 +1,542 @@
+# -*- coding: utf-8 -*-
+
+CSS = '''body { margin-left: 2%; margin-right: 2%; margin-top: 2%; margin-bottom: 2%; text-align: justify; }
+pre { font-size: x-small; }
+sml { font-size: small; }
+h1 { text-align: center; }
+h2 { text-align: center; }
+h3 { text-align: center; }
+h4 { text-align: center; }
+h5 { text-align: center; }
+h6 { text-align: center; }
+h7 { text-align: left; font-size: large; font-weight: bold; }
+.CI {
+    text-align:center;
+    margin-top:0px;
+    margin-bottom:0px;
+    padding:0px;
+    }
+.center   {text-align: center;}
+.cover    {text-align: center;}
+.full     {width: 100%; }
+.quarter  {width: 25%; }
+.smcap    {font-variant: small-caps;}
+.u        {text-decoration: underline;}
+.bold     {font-weight: bold;}
+'''
+
+MIMETYPE = '''application/epub+zip'''
+
+TITLE_HEADER = '''<?xml version="1.0" encoding="utf-8"?><html xmlns="http://www.w3.org/1999/xhtml" xmlns:xlink="http://www.w3.org/1999/xlink"><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+<title>%s - %s</title><link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/></head><body>
+<p><h7 id="lnks"><b><a id="StoryLink" href="%s">%s</a></b> by <b><a id="AuthorLink" href="%s">%s</a></b></h7></p>
+<table class="full">
+'''
+
+TITLE_ENTRY = '''<tr><td><b>%s</b></td><td>%s</td></tr>
+'''
+
+TITLE_FOOTER = '''</table>
+<p><b>Summary:</b><br />%s</p>
+</body></html>
+'''
+
+CONTAINER = '''<?xml version="1.0" encoding="utf-8"?>
+<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
+  <rootfiles>
+    <rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
+  </rootfiles>
+</container>
+'''
+
+CONTENT_START = '''<?xml version="1.0" encoding="utf-8"?>
+<package version="2.0" xmlns="http://www.idpf.org/2007/opf"
+         unique-identifier="fanficdownloader-uuid">
+ <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+           xmlns:dc="http://purl.org/dc/elements/1.1/"
+           xmlns:dcterms="http://purl.org/dc/terms/"
+           xmlns:opf="http://www.idpf.org/2007/opf"
+           xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata">
+   <dc:identifier id="fanficdownloader-uuid">BookID-Epub-%s</dc:identifier>
+   <dc:title>%s</dc:title> 
+   <dc:creator opf:role="aut">%s</dc:creator>
+   <dc:contributor opf:role="bkp">fanficdownloader [http://fanficdownloader.googlecode.com]</dc:contributor>
+   <dc:language>%s</dc:language> 
+   <dc:rights></dc:rights>
+   <dc:date opf:event="publication">%s</dc:date>
+   <dc:date opf:event="creation">%s</dc:date>
+   <dc:date opf:event="modification">%s</dc:date>
+   <meta name="calibre:timestamp" content="%s"/>
+   <dc:description>%s</dc:description>
+'''
+
+CONTENT_END_METADATA = '''   <dc:publisher>%s</dc:publisher> 
+   <dc:identifier id="BookId">%s</dc:identifier>
+   <dc:identifier opf:scheme="URL">%s</dc:identifier>
+   <dc:source>%s</dc:source>
+   <dc:type>FanFiction</dc:type>
+   <meta name="calibre:rating" content="%s"/>
+ </metadata>
+ <manifest>
+  <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml"/>
+  <item id="style" href="stylesheet.css" media-type="text/css" />
+'''
+
+CONTENT_SUBJECT = '''   <dc:subject>%s</dc:subject> 
+'''
+
+CONTENT_ITEM = '''  <item id="%s" href="%s" media-type="application/xhtml+xml" />
+'''
+
+CONTENT_END_MANIFEST = ''' </manifest>
+ <spine toc="ncx">
+'''
+
+CONTENT_ITEMREF = '''  <itemref idref="%s" />
+'''
+
+CONTENT_END = ''' </spine>
+</package>
+'''
+
+TOC_START = '''<?xml version="1.0" encoding="UTF-8"?>
+<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
+  <head>
+    <meta name="dtb:uid" content="%s"/>
+    <meta name="dtb:depth" content="1"/>
+    <meta name="dtb:totalPageCount" content="0"/>
+    <meta name="dtb:maxPageNumber" content="0"/>
+  </head>
+  <docTitle>
+    <text>%s</text>
+  </docTitle>
+  <navMap>
+'''
+
+TOC_ITEM = '''<navPoint id="%s" playOrder="%d">
+  <navLabel>
+    <text>%s</text>
+  </navLabel>
+  <content src="%s"/>
+</navPoint>
+'''
+
+TOC_END = '''</navMap>
+</ncx>
+'''
+
+XHTML_START = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>%s</title>
+<link href="stylesheet.css" type="text/css" rel="stylesheet" />
+</head>
+<body>
+<div>
+<h3>%s</h3>
+'''
+
+XHTML_END = '''</div>
+</body>
+</html>
+'''
+
+acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'b', 'big',
+      'blockquote', 'br', 'center', 'cite', 'code', 'col',
+      'colgroup', 'dd', 'del', 'dfn', 'dir', 'dl', 'dt', 'em',
+      'font', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'i', 
+      'ins', 'kbd', 'label', 'li', 'ol', 
+      'p', 'pre', 'q', 's', 'samp', 'small', 'span', 'strike',
+      'strong', 'sub', 'sup', 'u', 'ul']
+
+acceptable_attributes = ['href']
+
+# entity list from http://code.google.com/p/doctype/wiki/CharacterEntitiesConsistent
+entities = { '&aacute;' : 'á',
+             '&Aacute;' : 'Á',
+             '&Aacute' : 'Á',
+             '&aacute' : 'á',
+             '&acirc;' : 'â',
+             '&Acirc;' : 'Â',
+             '&Acirc' : 'Â',
+             '&acirc' : 'â',
+             '&acute;' : '´',
+             '&acute' : '´',
+             '&AElig;' : 'Æ',
+             '&aelig;' : 'æ',
+             '&AElig' : 'Æ',
+             '&aelig' : 'æ',
+             '&agrave;' : 'à',
+             '&Agrave;' : 'À',
+             '&Agrave' : 'À',
+             '&agrave' : 'à',
+             '&alefsym;' : 'ℵ',
+             '&alpha;' : 'α',
+             '&Alpha;' : 'Α',
+             '&amp;' : '&',
+             '&AMP;' : '&',
+             '&AMP' : '&',
+             '&amp' : '&',
+             '&and;' : '∧',
+             '&ang;' : '∠',
+             '&aring;' : 'å',
+             '&Aring;' : 'Å',
+             '&Aring' : 'Å',
+             '&aring' : 'å',
+             '&asymp;' : '≈',
+             '&atilde;' : 'ã',
+             '&Atilde;' : 'Ã',
+             '&Atilde' : 'Ã',
+             '&atilde' : 'ã',
+             '&auml;' : 'ä',
+             '&Auml;' : 'Ä',
+             '&Auml' : 'Ä',
+             '&auml' : 'ä',
+             '&bdquo;' : '„',
+             '&beta;' : 'β',
+             '&Beta;' : 'Β',
+             '&brvbar;' : '¦',
+             '&brvbar' : '¦',
+             '&bull;' : '•',
+             '&cap;' : '∩',
+             '&ccedil;' : 'ç',
+             '&Ccedil;' : 'Ç',
+             '&Ccedil' : 'Ç',
+             '&ccedil' : 'ç',
+             '&cedil;' : '¸',
+             '&cedil' : '¸',
+             '&cent;' : '¢',
+             '&cent' : '¢',
+             '&chi;' : 'χ',
+             '&Chi;' : 'Χ',
+             '&circ;' : 'ˆ',
+             '&clubs;' : '♣',
+             '&cong;' : '≅',
+             '&copy;' : '©',
+             '&COPY;' : '©',
+             '&COPY' : '©',
+             '&copy' : '©',
+             '&crarr;' : '↵',
+             '&cup;' : '∪',
+             '&curren;' : '¤',
+             '&curren' : '¤',
+             '&dagger;' : '†',
+             '&Dagger;' : '‡',
+             '&darr;' : '↓',
+             '&dArr;' : '⇓',
+             '&deg;' : '°',
+             '&deg' : '°',
+             '&delta;' : 'δ',
+             '&Delta;' : 'Δ',
+             '&diams;' : '♦',
+             '&divide;' : '÷',
+             '&divide' : '÷',
+             '&eacute;' : 'é',
+             '&Eacute;' : 'É',
+             '&Eacute' : 'É',
+             '&eacute' : 'é',
+             '&ecirc;' : 'ê',
+             '&Ecirc;' : 'Ê',
+             '&Ecirc' : 'Ê',
+             '&ecirc' : 'ê',
+             '&egrave;' : 'è',
+             '&Egrave;' : 'È',
+             '&Egrave' : 'È',
+             '&egrave' : 'è',
+             '&empty;' : '∅',
+             '&emsp;' : ' ',
+             '&ensp;' : ' ',
+             '&epsilon;' : 'ε',
+             '&Epsilon;' : 'Ε',
+             '&equiv;' : '≡',
+             '&eta;' : 'η',
+             '&Eta;' : 'Η',
+             '&eth;' : 'ð',
+             '&ETH;' : 'Ð',
+             '&ETH' : 'Ð',
+             '&eth' : 'ð',
+             '&euml;' : 'ë',
+             '&Euml;' : 'Ë',
+             '&Euml' : 'Ë',
+             '&euml' : 'ë',
+             '&euro;' : '€',
+             '&exist;' : '∃',
+             '&fnof;' : 'ƒ',
+             '&forall;' : '∀',
+             '&frac12;' : '½',
+             '&frac12' : '½',
+             '&frac14;' : '¼',
+             '&frac14' : '¼',
+             '&frac34;' : '¾',
+             '&frac34' : '¾',
+             '&frasl;' : '⁄',
+             '&gamma;' : 'γ',
+             '&Gamma;' : 'Γ',
+             '&ge;' : '≥',
+             '&gt;' : '>',
+             '&GT;' : '>',
+             '&GT' : '>',
+             '&gt' : '>',
+             '&harr;' : '↔',
+             '&hArr;' : '⇔',
+             '&hearts;' : '♥',
+             '&hellip;' : '…',
+             '&iacute;' : 'í',
+             '&Iacute;' : 'Í',
+             '&Iacute' : 'Í',
+             '&iacute' : 'í',
+             '&icirc;' : 'î',
+             '&Icirc;' : 'Î',
+             '&Icirc' : 'Î',
+             '&icirc' : 'î',
+             '&iexcl;' : '¡',
+             '&iexcl' : '¡',
+             '&igrave;' : 'ì',
+             '&Igrave;' : 'Ì',
+             '&Igrave' : 'Ì',
+             '&igrave' : 'ì',
+             '&image;' : 'ℑ',
+             '&infin;' : '∞',
+             '&int;' : '∫',
+             '&iota;' : 'ι',
+             '&Iota;' : 'Ι',
+             '&iquest;' : '¿',
+             '&iquest' : '¿',
+             '&isin;' : '∈',
+             '&iuml;' : 'ï',
+             '&Iuml;' : 'Ï',
+             '&Iuml' : 'Ï',
+             '&iuml' : 'ï',
+             '&kappa;' : 'κ',
+             '&Kappa;' : 'Κ',
+             '&lambda;' : 'λ',
+             '&Lambda;' : 'Λ',
+             '&laquo;' : '«',
+             '&laquo' : '«',
+             '&larr;' : '←',
+             '&lArr;' : '⇐',
+             '&lceil;' : '⌈',
+             '&ldquo;' : '“',
+             '&le;' : '≤',
+             '&lfloor;' : '⌊',
+             '&lowast;' : '∗',
+             '&loz;' : '◊',
+             '&lrm;' : '‎',
+             '&lsaquo;' : '‹',
+             '&lsquo;' : '‘',
+             '&lt;' : '<',
+             '&LT;' : '<',
+             '&LT' : '<',
+             '&lt' : '<',
+             '&macr;' : '¯',
+             '&macr' : '¯',
+             '&mdash;' : '—',
+             '&micro;' : 'µ',
+             '&micro' : 'µ',
+             '&middot;' : '·',
+             '&middot' : '·',
+             '&minus;' : '−',
+             '&mu;' : 'μ',
+             '&Mu;' : 'Μ',
+             '&nabla;' : '∇',
+             '&nbsp;' : ' ',
+             '&nbsp' : ' ',
+             '&ndash;' : '–',
+             '&ne;' : '≠',
+             '&ni;' : '∋',
+             '&not;' : '¬',
+             '&not' : '¬',
+             '&notin;' : '∉',
+             '&nsub;' : '⊄',
+             '&ntilde;' : 'ñ',
+             '&Ntilde;' : 'Ñ',
+             '&Ntilde' : 'Ñ',
+             '&ntilde' : 'ñ',
+             '&nu;' : 'ν',
+             '&Nu;' : 'Ν',
+             '&oacute;' : 'ó',
+             '&Oacute;' : 'Ó',
+             '&Oacute' : 'Ó',
+             '&oacute' : 'ó',
+             '&ocirc;' : 'ô',
+             '&Ocirc;' : 'Ô',
+             '&Ocirc' : 'Ô',
+             '&ocirc' : 'ô',
+             '&OElig;' : 'Œ',
+             '&oelig;' : 'œ',
+             '&ograve;' : 'ò',
+             '&Ograve;' : 'Ò',
+             '&Ograve' : 'Ò',
+             '&ograve' : 'ò',
+             '&oline;' : '‾',
+             '&omega;' : 'ω',
+             '&Omega;' : 'Ω',
+             '&omicron;' : 'ο',
+             '&Omicron;' : 'Ο',
+             '&oplus;' : '⊕',
+             '&or;' : '∨',
+             '&ordf;' : 'ª',
+             '&ordf' : 'ª',
+             '&ordm;' : 'º',
+             '&ordm' : 'º',
+             '&oslash;' : 'ø',
+             '&Oslash;' : 'Ø',
+             '&Oslash' : 'Ø',
+             '&oslash' : 'ø',
+             '&otilde;' : 'õ',
+             '&Otilde;' : 'Õ',
+             '&Otilde' : 'Õ',
+             '&otilde' : 'õ',
+             '&otimes;' : '⊗',
+             '&ouml;' : 'ö',
+             '&Ouml;' : 'Ö',
+             '&Ouml' : 'Ö',
+             '&ouml' : 'ö',
+             '&para;' : '¶',
+             '&para' : '¶',
+             '&part;' : '∂',
+             '&permil;' : '‰',
+             '&perp;' : '⊥',
+             '&phi;' : 'φ',
+             '&Phi;' : 'Φ',
+             '&pi;' : 'π',
+             '&Pi;' : 'Π',
+             '&piv;' : 'ϖ',
+             '&plusmn;' : '±',
+             '&plusmn' : '±',
+             '&pound;' : '£',
+             '&pound' : '£',
+             '&prime;' : '′',
+             '&Prime;' : '″',
+             '&prod;' : '∏',
+             '&prop;' : '∝',
+             '&psi;' : 'ψ',
+             '&Psi;' : 'Ψ',
+             '&quot;' : '"',
+             '&QUOT;' : '"',
+             '&QUOT' : '"',
+             '&quot' : '"',
+             '&radic;' : '√',
+             '&raquo;' : '»',
+             '&raquo' : '»',
+             '&rarr;' : '→',
+             '&rArr;' : '⇒',
+             '&rceil;' : '⌉',
+             '&rdquo;' : '”',
+             '&real;' : 'ℜ',
+             '&reg;' : '®',
+             '&REG;' : '®',
+             '&REG' : '®',
+             '&reg' : '®',
+             '&rfloor;' : '⌋',
+             '&rho;' : 'ρ',
+             '&Rho;' : 'Ρ',
+             '&rlm;' : '‏',
+             '&rsaquo;' : '›',
+             '&rsquo;' : '’',
+             '&sbquo;' : '‚',
+             '&scaron;' : 'š',
+             '&Scaron;' : 'Š',
+             '&sdot;' : '⋅',
+             '&sect;' : '§',
+             '&sect' : '§',
+             '&shy;' : '­', # strange optional hyphenation control character, not just a dash
+             '&shy' : '­',
+             '&sigma;' : 'σ',
+             '&Sigma;' : 'Σ',
+             '&sigmaf;' : 'ς',
+             '&sim;' : '∼',
+             '&spades;' : '♠',
+             '&sub;' : '⊂',
+             '&sube;' : '⊆',
+             '&sum;' : '∑',
+             '&sup1;' : '¹',
+             '&sup1' : '¹',
+             '&sup2;' : '²',
+             '&sup2' : '²',
+             '&sup3;' : '³',
+             '&sup3' : '³',
+             '&sup;' : '⊃',
+             '&supe;' : '⊇',
+             '&szlig;' : 'ß',
+             '&szlig' : 'ß',
+             '&tau;' : 'τ',
+             '&Tau;' : 'Τ',
+             '&there4;' : '∴',
+             '&theta;' : 'θ',
+             '&Theta;' : 'Θ',
+             '&thetasym;' : 'ϑ',
+             '&thinsp;' : ' ',
+             '&thorn;' : 'þ',
+             '&THORN;' : 'Þ',
+             '&THORN' : 'Þ',
+             '&thorn' : 'þ',
+             '&tilde;' : '˜',
+             '&times;' : '×',
+             '&times' : '×',
+             '&trade;' : '™',
+             '&uacute;' : 'ú',
+             '&Uacute;' : 'Ú',
+             '&Uacute' : 'Ú',
+             '&uacute' : 'ú',
+             '&uarr;' : '↑',
+             '&uArr;' : '⇑',
+             '&ucirc;' : 'û',
+             '&Ucirc;' : 'Û',
+             '&Ucirc' : 'Û',
+             '&ucirc' : 'û',
+             '&ugrave;' : 'ù',
+             '&Ugrave;' : 'Ù',
+             '&Ugrave' : 'Ù',
+             '&ugrave' : 'ù',
+             '&uml;' : '¨',
+             '&uml' : '¨',
+             '&upsih;' : 'ϒ',
+             '&upsilon;' : 'υ',
+             '&Upsilon;' : 'Υ',
+             '&uuml;' : 'ü',
+             '&Uuml;' : 'Ü',
+             '&Uuml' : 'Ü',
+             '&uuml' : 'ü',
+             '&weierp;' : '℘',
+             '&xi;' : 'ξ',
+             '&Xi;' : 'Ξ',
+             '&yacute;' : 'ý',
+             '&Yacute;' : 'Ý',
+             '&Yacute' : 'Ý',
+             '&yacute' : 'ý',
+             '&yen;' : '¥',
+             '&yen' : '¥',
+             '&yuml;' : 'ÿ',
+             '&Yuml;' : 'Ÿ',
+             '&yuml' : 'ÿ',
+             '&zeta;' : 'ζ',
+             '&Zeta;' : 'Ζ',
+             '&zwj;' : '‍',  # strange spacing control character, not just a space
+             '&zwnj;' : '‌',  # strange spacing control character, not just a space
+             }
+
+FB2_PROLOGUE = '<FictionBook>'
+FB2_DESCRIPTION = '''<description>
+<title-info>
+  <genre>fanfiction</genre>
+  <author>
+  <first-name></first-name>
+  <middle-name></middle-name>
+  <last-name>%s</last-name>
+  </author>
+  <book-title>%s</book-title>
+  <lang>eng</lang>
+</title-info>
+<document-info>
+  <author>
+  <nickname>sgzmd</nickname>
+  </author>
+<date value="%s">%s</date>
+<id>sgzmd_%s</id>
+<version>2.0</version>
+</document-info>
+</description>'''
+
+HTML_ESC_Definitions = 'HTML_Escape.def'

cron.yaml

@@ -1,10 +0,0 @@
-cron:
-- description: cleanup job
-  url: /r3m0v3r
-  schedule: every 2 hours
-
-# There's a bug in the Python 2.7 runtime that prevents this from
-# working properly.  In theory, there should never be orphans anyway.
-#- description: orphan cleanup job
-#  url: /r3m0v3rOrphans
-#  schedule: every 4 hours

css/index.css

@@ -1,73 +0,0 @@
-body
-{
-	font: 0.9em "Helvetica Neue", Arial, Helvetica, Geneva, sans-serif;
-}
-
-#main
-{
-	width: 60%;
-	margin-left: 20%;
-	background-color: #dae6ff;
-	padding: 2em;
-}
-
-#greeting
-{
-#	margin-bottom: 1em;
-	border-color: #efefef;
-}
-
-
-
-#logpassword:hover, #logpasswordtable:hover, #urlbox:hover, #typebox:hover, #helpbox:hover, #yourfile:hover
-{
-	border: thin solid #fffeff;	
-}
-
-h1
-{
-	text-decoration: none;
-}
-
-#logpasswordtable
-{
-	padding: 1em;
-}
-
-#logpassword, #logpasswordtable {
-//  display: none;
-}
-
-#urlbox, #typebox, #logpasswordtable, #logpassword, #helpbox, #yourfile
-{
-	margin: 1em;
-	padding: 1em;
-	border: thin dotted #fffeff;	
-}
-
-div.field
-{
-	margin-bottom: 0.5em;
-}
-
-#submitbtn
-{
-	padding: 1em;
-}
-
-#typelabel
-{
-}
-
-#typeoptions
-{
-	margin-top: 0.5em;
-}
-
-#error
-{
-	color: #f00;
-}
-.recent {
-    font-size: large;
-}

delete_fic.py

@@ -1,59 +0,0 @@
-import os
-import cgi
-import sys
-import logging
-import traceback
-import StringIO
-
-from google.appengine.api import users
-from google.appengine.ext import webapp
-from google.appengine.ext.webapp import util
-
-from fanficdownloader.downaloder import *
-from fanficdownloader.ffnet import *
-from fanficdownloader.output import *
-
-from google.appengine.ext import db
-
-from fanficdownloader.zipdir import *
-
-from ffstorage import *
-
-def create_mac(user, fic_id, fic_url):
-  return str(abs(hash(user)+hash(fic_id)))+str(abs(hash(fic_url)))
-  
-def check_mac(user, fic_id, fic_url, mac):
-  return (create_mac(user, fic_id, fic_url) == mac)
-
-def create_mac_for_fic(user, fic_id):
-  key = db.Key(fic_id)
-	fanfic = db.get(key)
-	if fanfic.user != user:
-	  return None
-	else:
-	  return create_mac(user, key, fanfic.url)
-
-class DeleteFicHandler(webapp.RequestHandler):
-	def get(self):
-		user = users.get_current_user()
-		if not user:
-			self.redirect('/login')
-
-    fic_id = self.request.get('fic_id')
-    fic_mac = self.request.get('key_id')
-    
-    actual_mac = create_mac_for_fic(user, fic_id)
-    if actual_mac != fic_mac:
-      self.response.out.write("Ooops")
-    else:
-      key = db.Key(fic_id)
-    	fanfic = db.get(key)
-      fanfic.delete()
-      self.redirect('/recent')
-    
-
-		fics = db.GqlQuery("Select * From DownloadedFanfic WHERE user = :1", user)
-		template_values = dict(fics = fics, nickname = user.nickname())
-		path = os.path.join(os.path.dirname(__file__), 'recent.html')
-		self.response.out.write(template.render(path, template_values))
-	

downloader.py

@@ -1,247 +1,193 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import sys, os
-from os.path import normpath, expanduser, isfile, join
-from StringIO import StringIO
-from optparse import OptionParser      
-import getpass
-import string
-import ConfigParser
-from subprocess import call
-
-import logging
-if sys.version_info >= (2, 7):
-    # suppresses default logger.  Logging is setup in fanficdownload/__init__.py so it works in calibre, too.
-    rootlogger = logging.getLogger()
-    loghandler=logging.NullHandler()
-    loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
-    rootlogger.addHandler(loghandler)
-
-from fanficdownloader import adapters,writers,exceptions
-from fanficdownloader.configurable import Configuration
-from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-from fanficdownloader.geturls import get_urls_from_page
-
-if sys.version_info < (2, 5):
-    print "This program requires Python 2.5 or newer."
-    sys.exit(1)
-
-def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
-    writer = writers.getWriter(writeformat,config,adapter)
-    writer.writeStory(outstream=outstream,metaonly=metaonly)
-    output_filename=writer.getOutputFileName()
-    del writer
-    return output_filename
-
-def main():
-   # read in args, anything starting with -- will be treated as --<varible>=<value>
-   usage = "usage: %prog [options] storyurl"
-   parser = OptionParser(usage)
-   parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), text or html", metavar="FORMAT")
-   parser.add_option("-c", "--config",
-                     action="append", dest="configfile", default=None,
-                     help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG")
-   parser.add_option("-b", "--begin", dest="begin", default=None,
-                     help="Begin with Chapter START", metavar="START")
-   parser.add_option("-e", "--end", dest="end", default=None,
-                     help="End with Chapter END", metavar="END")
-   parser.add_option("-o", "--option",
-                     action="append", dest="options",
-                     help="set an option NAME=VALUE", metavar="NAME=VALUE")
-   parser.add_option("-m", "--meta-only",
-                     action="store_true", dest="metaonly",
-                     help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
-   parser.add_option("-u", "--update-epub",
-                     action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
-   parser.add_option("--update-cover",
-                     action="store_true", dest="updatecover",
-                     help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
-   parser.add_option("--force",
-                     action="store_true", dest="force",
-                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
-   parser.add_option("-l", "--list",
-                     action="store_true", dest="list",
-                     help="Get list of valid story URLs from page given.",)
-   parser.add_option("-d", "--debug",
-                     action="store_true", dest="debug",
-                     help="Show debug output while downloading.",)
-   
-   (options, args) = parser.parse_args()
-
-   if not options.debug:
-       logger = logging.getLogger("fanficdownloader")
-       logger.setLevel(logging.INFO)
-   
-   if len(args) != 1:
-       parser.error("incorrect number of arguments")
-
-   if options.update and options.format != 'epub':
-       parser.error("-u/--update-epub only works with epub")
-
-   ## Attempt to update an existing epub.
-   if options.update:
-       (url,chaptercount) = get_dcsource_chaptercount(args[0])
-       print "Updating %s, URL: %s" % (args[0],url)
-       output_filename = args[0]
-   else:
-       url = args[0]
-           
-   configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
-
-   conflist = []
-   homepath = join(expanduser("~"),".fanficdownloader")
-   
-   if isfile(join(homepath,"defaults.ini")):
-       conflist.append(join(homepath,"defaults.ini"))
-   if isfile("defaults.ini"):
-       conflist.append("defaults.ini")
-       
-   if isfile(join(homepath,"personal.ini")):
-       conflist.append(join(homepath,"personal.ini"))
-   if isfile("personal.ini"):
-       conflist.append("personal.ini")
-       
-   if options.configfile:
-       conflist.extend(options.configfile)
-      
-   logging.debug('reading %s config file(s), if present'%conflist)
-   configuration.read(conflist)
-
-   try:
-       configuration.add_section("overrides")
-   except ConfigParser.DuplicateSectionError:
-       pass
-
-   if options.force:
-       configuration.set("overrides","always_overwrite","true")
-
-   if options.update:
-       configuration.set("overrides","output_filename",args[0])
-       
-   if options.update and not options.updatecover:
-       configuration.set("overrides","never_make_cover","true")
-
-   # images only for epub, even if the user mistakenly turned it
-   # on else where.
-   if options.format not in ("epub","html"):
-       configuration.set("overrides","include_images","false")
-       
-   if options.options:
-       for opt in options.options:
-           (var,val) = opt.split('=')
-           configuration.set("overrides",var,val)
-
-   if options.list:
-       retlist = get_urls_from_page(args[0], configuration)
-       print "\n".join(retlist)
-               
-       return
-
-   try:
-
-       adapter = adapters.getAdapter(configuration,url)
-
-       ## Check for include_images and absence of PIL, give warning.
-       if adapter.getConfig('include_images'):
-           try:
-               from calibre.utils.magick import Image
-               logging.debug("Using calibre.utils.magick")
-           except:
-               try:
-                   import Image
-                   logging.debug("Using PIL")
-               except:
-                   print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
-                   if not sys.stdin.readline().strip().lower().startswith('y'):
-                       return
-               
-
-       ## three tries, that's enough if both user/pass & is_adult needed,
-       ## or a couple tries of one or the other
-       for x in range(0,2):
-           try:
-               adapter.getStoryMetadataOnly()
-           except exceptions.FailedToLogin, f:
-               if f.passwdonly:
-                   print "Story requires a password."
-               else:
-                   print "Login Failed, Need Username/Password."
-                   sys.stdout.write("Username: ")
-                   adapter.username = sys.stdin.readline().strip()
-               adapter.password = getpass.getpass(prompt='Password: ')
-               #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
-           except exceptions.AdultCheckRequired:
-               print "Please confirm you are an adult in your locale: (y/n)?"
-               if sys.stdin.readline().strip().lower().startswith('y'):
-                   adapter.is_adult=True
-
-       if options.update and not options.force:
-           urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
-           
-           if chaptercount == urlchaptercount and not options.metaonly:
-               print "%s already contains %d chapters." % (args[0],chaptercount)
-           elif chaptercount > urlchaptercount:
-               print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
-           else:
-               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
-               if not options.metaonly:
-
-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(args[0])
-
-                   writeStory(configuration,adapter,"epub")
-                   
-       else:
-           # regular download
-           if options.metaonly:
-               print adapter.getStoryMetadataOnly()
-           
-           adapter.setChaptersRange(options.begin,options.end)
-           
-           output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
-       
-       if not options.metaonly and adapter.getConfig("post_process_cmd"):
-           metadata = adapter.story.metadata
-           metadata['output_filename']=output_filename
-           call(string.Template(adapter.getConfig("post_process_cmd"))
-                .substitute(metadata), shell=True)
-           
-       del adapter
-   
-   except exceptions.InvalidStoryURL, isu:
-       print isu
-   except exceptions.StoryDoesNotExist, dne:
-       print dne
-   except exceptions.UnknownSite, us:
-       print us
-
-if __name__ == "__main__":
-    #import time
-    #start = time.time()
-    main()
-    #print("Total time seconds:%f"%(time.time()-start))
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import sys
+import shutil
+import os.path
+import getpass
+import logging
+import urllib as u
+import pprint as pp
+import urllib2 as u2
+import urlparse as up
+import BeautifulSoup as bs
+import htmlentitydefs as hdefs
+
+import zipdir
+
+import output
+import adapter
+from adapter import StoryArchivedAlready
+from adapter import StoryDoesNotExist
+from adapter import FailedToDownload
+from adapter import InvalidStoryURL
+from adapter import LoginRequiredException
+import ffnet
+import fpcom
+import ficwad
+import fictionalley
+import hpfiction
+import twilighted
+import potionsNsnitches
+import mediaminer
+
+import time
+
+class FanficLoader:
+	'''A controller class which handles the interaction between various specific downloaders and writers'''
+	booksDirectory = "books"
+	standAlone = False
+	
+	def __init__(self, adapter, writerClass, quiet = False, inmemory = False, compress=True, overwrite=False):
+		self.adapter = adapter
+		self.writerClass = writerClass
+		self.quiet = quiet
+		self.inmemory = inmemory
+		self.compress = compress
+		self.badLogin = False
+		self.overWrite = overwrite
+			
+	def getBooksDirectory(self):
+		return self.booksDirectory
+
+	def setBooksDirectory(self, bd):
+		self.booksDirectory = bd
+		return self.booksDirectory
+	
+	def getStandAlone(self):
+		return self.standAlone
+
+	def setStandAlone(self, sa):
+		self.standAlone = sa
+		return self.standAlone
+	
+	def getAdapter():
+		return self.adapter
+	
+	def download(self):
+		logging.debug("Trying to download the story")
+		if self.adapter.requiresLogin():
+			logging.debug("Story requires login")
+			if not self.adapter.performLogin():
+				logging.debug("Login/password problem")
+				self.badLogin = True
+				raise adapter.LoginRequiredException(self.adapter.url)
+		
+		urls = self.adapter.extractIndividualUrls()
+
+		logging.debug("self.writerClass=%s" % self.writerClass)
+		if self.standAlone and not self.inmemory:
+			s = self.adapter.getOutputFileName(self.booksDirectory, self.writerClass.getFormatExt())
+			logging.debug("Always overwrite? %s" % self.overWrite)
+			if not self.overWrite:
+				logging.debug("Checking if current archive of the story exists.  Filename=%s" % s)
+				if not zipdir.checkNewer ( s, self.adapter.getStoryUpdated() ):
+					raise StoryArchivedAlready("A Current archive file \"" + s + "\" already exists!  Skipping!")
+		else:
+			logging.debug("Do not check for existance of archive file.")
+
+		self.writer = self.writerClass(self.booksDirectory, self.adapter, inmemory=self.inmemory, compress=self.compress)
+		
+		i = 1
+		for u,n in urls:
+			if not self.quiet:
+				print('Downloading chapter %d/%d' % (i, len(urls)))
+			text = self.adapter.getText(u)
+			self.writer.writeChapter(i, n, text)
+			i = i+1
+			# time.sleep(2)
+			
+		self.writer.finalise()
+		
+		if self.inmemory:
+			self.name = self.writer.name
+			return self.writer.output.getvalue()
+	
+
+if __name__ == '__main__':
+	logging.basicConfig(level=logging.DEBUG)
+	argvlen = len(sys.argv)
+	url = None
+	bookFormat = 'epub'
+	if argvlen > 1:
+		url = sys.argv[1]
+	if argvlen > 2:
+		bookFormat = sys.argv[2]
+	
+	if url is None: 
+		print >> sys.stderr, "Usage: downloader.py URL Type"
+		sys.exit(-1)
+		
+	if type(url) is unicode:
+		print('URL is unicode')
+		url = url.encode('latin1')
+	url = url.strip()
+	adapter = None
+	writerClass = None
+	
+	if url.find('fanficauthors') != -1:
+		print >> sys.stderr, "fanficauthors.net already provides ebooks"
+		sys.exit(0)
+	elif url.find('fictionalley') != -1:
+		adapter = fictionalley.FictionAlley(url)
+	elif url.find('ficwad') != -1:
+		adapter = ficwad.FicWad(url)
+	elif url.find('fanfiction.net') != -1:
+		adapter = ffnet.FFNet(url)
+	elif url.find('fictionpress.com') != -1:
+		adapter = fpcom.FPCom(url)
+	elif url.find('harrypotterfanfiction.com') != -1:
+		adapter = hpfiction.HPFiction(url)
+	elif url.find('twilighted.net') != -1:
+		adapter = twilighted.Twilighted(url)
+	elif url.find('potionsandsnitches.net') != -1:
+		adapter = potionsNsnitches.PotionsNSnitches(url)
+	elif url.find('mediaminer.org') != -1:
+		adapter = mediaminer.MediaMiner(url)
+	else:
+		print >> sys.stderr, "Oi! I can haz not appropriate adapter for URL %s!" % url
+		sys.exit(1)
+
+	if bookFormat == 'epub':
+		writerClass = output.EPubFanficWriter
+	elif bookFormat == 'html':
+		writerClass = output.HTMLWriter
+	
+	if adapter.requiresLogin(url):
+		print("Meow, URL %s requires you to haz been logged in! Please can I haz this datas?" % url)
+		sys.stdout.write("Can I haz ur login? ")
+		login = sys.stdin.readline().strip()
+		password = getpass.getpass(prompt='Can I haz ur password? ')
+		print("Login: `%s`, Password: `%s`" % (login, password))
+		
+		adapter.setLogin(login)
+		adapter.setPassword(password)
+		
+	
+	loader = FanficLoader(adapter, writerClass)
+	loader.setStandAlone(True)
+
+	try:
+		loader.download()
+	except FailedToDownload, ftd:
+		print >> sys.stderr, str(ftd)
+		sys.exit(2)		# Error Downloading
+	except InvalidStoryURL, isu:
+		print >> sys.stderr, str(isu)
+		sys.exit(3)		# Unknown Error
+	except StoryArchivedAlready, se:
+		print >> sys.stderr, str(se)
+		sys.exit(10)	# Skipped
+	except StoryDoesNotExist, sdne:
+		print >> sys.stderr, str(sdne)
+		sys.exit(20) 	# Missing
+	except LoginRequiredException, lre:
+		print >> sys.stderr, str(lre)
+		sys.exit(30) 	# Missing
+	except Exception, e:
+		print >> sys.stderr, str(e)
+		sys.exit(99)		# Unknown Error
+	
+	sys.exit(0)
+	

editconfig.html

@@ -1,89 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">
-<html>
-  <head>
-    <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
-    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-    <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
-    <script type="text/javascript">
-
-      var _gaq = _gaq || [];
-      _gaq.push(['_setAccount', 'UA-12136939-1']);
-      _gaq.push(['_trackPageview']);
-
-      (function() {
-      var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
-      ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
-      var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
-      })();
-
-    </script>
-  </head>
-  <body>
-    <div id='main' style="width: 80%; margin-left: 10%;">
-      <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
-      </h1>
-
-      <div style="text-align: center">
-        <script type="text/javascript"><!--
-                                          google_ad_client = "ca-pub-0320924304307555";
-                                          /* Standard */
-                                          google_ad_slot = "8974025478";
-                                          google_ad_width = 468;
-                                          google_ad_height = 60;
-                                          //-->
-        </script>
-        <script type="text/javascript"
-                src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
-        </script>
-      </div>
-
-      <form action="/editconfig" method="post">
-	<input type="hidden" name="update" value="true" />
-	<div id='logpasswordtable'>
-	  <h3>Edit Config</h3>
-	  <div id='logpassword'>
-	    Editing configuration for {{ nickname }}.
-	  </div>
-	  <div class='fieldandlabel'>
-	    <textarea name="config" style="width: 100%; height: 200px;" wrap='off'>{{ config }}</textarea>
-	  </div>
-	</div>
-	
-	<div id='submitbtn'>
-	  <input type="submit" value="Save">
-	</div>
-      </form>
-
-      <div>
-	<h3>Default System configuration</h3>
-<pre>
-{{ defaultsini }}
-</pre>
-      </div>
-      
-      <div style='text-align: center'>
-	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
-	     alt="Powered by Google App Engine" />
-	<br/><br/>
-	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
-	Copyright &copy; Fanficdownloader team
-      </div>
-
-      <div style="margin-top: 1em; text-align: center'">
-        <script type="text/javascript"><!--
-                                          google_ad_client = "pub-2027714004231956";
-                                          /* FFD */
-                                          google_ad_slot = "7330682770";
-                                          google_ad_width = 468;
-                                          google_ad_height = 60;
-                                          //-->
-        </script>
-        <script type="text/javascript"
-                src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
-        </script>
-      </div>
-    </div>
-  </body>
-</html>

epubmerge.py

@@ -1,25 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-# epubmerge.py 1.0
-
-# Copyright 2011, Jim Miller
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#   http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-    
-if __name__ == "__main__":
-    print('''
-The this utility has been split out into it's own project. 
-See: http://code.google.com/p/epubmerge/
-...for a CLI epubmerge.py program and calibre plugin.
-''')

example.ini

@@ -1,103 +0,0 @@
-## This is an example of what your personal configuration might look
-## like.  Uncomment options by removing the '#' in front of them.
-
-[defaults]
-## Some sites also require the user to confirm they are adult for
-## adult content.  Uncomment by removing '#' in front of is_adult.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#is_adult:true
-
-## Don't like the numbers at the start of chapter titles on some
-## sites?  You can use strip_chapter_numbers to strip them off.  Just
-## want to make them all look the same?  Strip them off, then add them
-## back on with add_chapter_numbers.  Don't like the way it strips
-## numbers or adds them back?  See chapter_title_strip_pattern and
-## chapter_title_add_pattern.
-#strip_chapter_numbers:true
-#add_chapter_numbers:true
-
-[epub]
-## include images from img tags in the body and summary of stories.
-## Images will be converted to jpg for size if possible.  Images work
-## in epub format only.  To get mobi or other format with images,
-## download as epub and use Calibre to convert.
-#include_images:true
-
-## If not set, the summary will have all html stripped for safety.
-## Both this and include_images must be true to get images in the
-## summary.
-#keep_summary_html:true
-
-## If set, the first image found will be made the cover image.  If
-## keep_summary_html is true, any images in summary will be before any
-## in chapters.
-#make_firstimage_cover:true
-
-## Resize images down to width, height, preserving aspect ratio.
-## Nook size, with margin.
-#image_max_size: 580, 725
-
-## Change image to grayscale, if graphics library allows, to save
-## space.
-#grayscale_images: false
-
-
-## Most common, I expect will be using this to save username/passwords
-## for different sites.  Here are a few examples.  See defaults.ini
-## for the full list.
-
-[www.twilighted.net]
-#username:YourPenname
-#password:YourPassword
-## default is false
-#collect_series: true
-
-[www.ficwad.com]
-#username:YourUsername
-#password:YourPassword
-
-[www.twiwrite.net]
-#username:YourName
-#password:yourpassword
-## default is false
-#collect_series: true
-
-[www.adastrafanfic.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.
-#is_adult:true
-
-[www.thewriterscoffeeshop.com]
-#username:YourName
-#password:yourpassword
-#is_adult:true
-## default is false
-#collect_series: true
-
-[www.fictionalley.org]
-#is_adult:true
-
-[www.harrypotterfanfiction.com]
-#is_adult:true
-
-[www.fimfiction.net]
-#is_adult:true
-#fail_on_password: false
-
-[www.tthfanfic.org]
-#is_adult:true
-## tth is a little unusual--it doesn't require user/pass, but the site
-## keeps track of which chapters you've read and won't send another
-## update until it thinks you're up to date.  This way, on download,
-## it thinks you're up to date.
-#username:YourName
-#password:yourpassword
-
-
-## This section will override anything in the system defaults or other
-## sections here.
-[overrides]
-## default varies by site.  Set true here to force all sites to
-## collect series.
-#collect_series: true

fanficdownloader/__init__.py

@@ -1,16 +0,0 @@
-# -*- coding: utf-8 -*-
-
-try:
-    # just a way to switch between web service and CLI/PI
-    import google.appengine.api 
-except:
-    import sys
-    if sys.version_info >= (2, 7):
-        import logging
-        logger = logging.getLogger(__name__)
-        loghandler=logging.StreamHandler()
-        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-        logger.addHandler(loghandler)
-        loghandler.setLevel(logging.DEBUG)
-        logger.setLevel(logging.DEBUG)
-

fanficdownloader/adapters/__init__.py

@@ -1,179 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob, types
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-logger = logging.getLogger(__name__)
-
-from .. import exceptions as exceptions
-
-## must import each adapter here.
-
-import adapter_test1
-import adapter_fanfictionnet
-import adapter_castlefansorg
-import adapter_fictionalleyorg
-import adapter_fictionpresscom
-import adapter_ficwadcom
-import adapter_fimfictionnet
-import adapter_harrypotterfanfictioncom
-import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
-import adapter_tenhawkpresentscom
-import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
-import adapter_tthfanficorg
-import adapter_twilightednet
-import adapter_twiwritenet
-import adapter_whoficcom
-import adapter_siyecouk
-import adapter_archiveofourownorg
-import adapter_ficbooknet
-import adapter_portkeyorg
-import adapter_mugglenetcom
-import adapter_hpfandomnet
-import adapter_thequidditchpitchorg
-import adapter_nfacommunitycom
-import adapter_midnightwhispersca
-import adapter_ksarchivecom
-import adapter_archiveskyehawkecom
-import adapter_squidgeorgpeja
-import adapter_libraryofmoriacom
-import adapter_wraithbaitcom
-import adapter_checkmatedcom
-import adapter_chaossycophanthexcom
-import adapter_dramioneorg
-import adapter_erosnsapphosycophanthexcom
-import adapter_lumossycophanthexcom
-import adapter_occlumencysycophanthexcom
-import adapter_phoenixsongnet
-import adapter_walkingtheplankorg
-import adapter_ashwindersycophanthexcom
-import adapter_thehexfilesnet
-import adapter_dokugacom
-import adapter_iketernalnet
-import adapter_onedirectionfanfictioncom
-import adapter_prisonbreakficnet
-import adapter_storiesofardacom
-import adapter_yourfanfictioncom
-import adapter_samdeanarchivenu
-import adapter_destinysgatewaycom
-import adapter_ncisfictioncom
-import adapter_stargateatlantisorg
-import adapter_thealphagatecom
-import adapter_fanfiktionde
-import adapter_ponyfictionarchivenet
-import adapter_sg1heliopoliscom
-import adapter_ncisficcom
-import adapter_nationallibrarynet
-import adapter_themasquenet
-import adapter_pretendercentrecom
-import adapter_darksolaceorg
-import adapter_finestoriescom
-import adapter_hpfanficarchivecom
-import adapter_svufictioncom
-import adapter_twilightarchivescom
-import adapter_wizardtalesnet
-import adapter_nhamagicalworldsus
-import adapter_hlfictionnet
-import adapter_grangerenchantedcom
-import adapter_dracoandginnycom
-import adapter_scarvesandcoffeenet
-import adapter_thepetulantpoetesscom
-import adapter_wolverineandroguecom
-import adapter_sinfuldesireorg
-import adapter_merlinficdtwinscouk
-import adapter_thehookupzonenet
-import adapter_bloodtiesfancom
-import adapter_qafficcom
-
-## This bit of complexity allows adapters to be added by just adding
-## importing.  It eliminates the long if/else clauses we used to need
-## to pick out the adapter.
-    
-## List of registered site adapters.
-__class_list = []
-
-def imports():
-    for name, val in globals().items():
-        if isinstance(val, types.ModuleType):
-            yield val.__name__
-
-for x in imports():
-    if "fanficdownloader.adapters.adapter_" in x:
-        #print x
-        __class_list.append(sys.modules[x].getClass())
-
-def getAdapter(config,url):
-
-    logger.debug("trying url:"+url)
-    (cls,fixedurl) = getClassFor(url)
-    logger.debug("fixedurl:"+fixedurl)
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getConfigSections():
-    return [cls.getConfigSection() for cls in __class_list]
-
-def getConfigSectionFor(url):
-    (cls,fixedurl) = getClassFor(url)
-    if cls:
-        return cls.getConfigSection()
-    
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getClassFor(url):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    cls = getClassFromList(domain)
-    if not cls and domain.startswith("www."):
-        domain = domain.replace("www.","")
-        logger.debug("trying site:without www: "+domain)
-        cls = getClassFromList(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
-    if not cls:
-        logger.debug("trying site:www."+domain)
-        cls = getClassFromList("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-        
-    return (cls,fixedurl)
-    
-def getClassFromList(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -1,228 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','aaff')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'www.adastrafanfic.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            addurl = "&warning=5"
-        else:
-            addurl=""
-            
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):]
-        
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        ## <meta name='description' content='&lt;p&gt;Description&lt;/p&gt; ...' >
-        ## Summary, strangely, is in the content attr of a <meta name='description'> tag
-        ## which is escaped HTML.  Unfortunately, we can't use it because they don't
-        ## escape (') chars in the desc, breakin the tag.
-        #meta_desc = soup.find('meta',{'name':'description'})
-        #metasoup = bs.BeautifulStoneSoup(meta_desc['content'])
-        #self.story.setMetadata('description',stripHTML(metasoup))
-
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-        
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ''
-                while value and not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                # sometimes poorly formated desc (<p> w/o </p>) leads
-                # to all labels being included.
-                svalue=svalue[:svalue.find('<span class="label">')]
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):]
-        
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
-
-def getClass():
-    return AdAstraFanficComSiteAdapter
-

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -1,344 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ArchiveOfOurOwnOrgAdapter
-
-
-logger = logging.getLogger(__name__)
-
-class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-							   
-							   
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-            # normalized story URL.
-            self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ao3')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y-%b-%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'archiveofourown.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
-
-    def getSiteURLPattern(self):
-        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)"
-        
-    ## Login
-    def needToLoginCheck(self, data):
-        if 'This work is only available to registered users of the Archive.' in data \
-                or "The password or user name you entered doesn't match our records" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url, data):
-
-        params = {}
-        if self.password:
-            params['user_session[login]'] = self.username
-            params['user_session[password]'] = self.password
-        else:
-            params['user_session[login]'] = self.getConfig("username")
-            params['user_session[password]'] = self.getConfig("password")
-        params['user_session[remember_me]'] = '1'
-        params['commit'] = 'Log in'
-        #params['utf8'] = u'✓'#u'\x2713' # gets along with out it, and it confuses the encoder.
-        params['authenticity_token'] = data.split('input name="authenticity_token" type="hidden" value="')[1].split('" /></div>')[0]
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/user_sessions'
-        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['user_session[login]']))
-    
-        d = self._postUrl(loginUrl, params)
-        #logger.info(d)
-    
-        if "Successfully logged in" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['user_session[login]']))
-            raise exceptions.FailedToLogin(url,params['user_session[login]'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            addurl = "?view_adult=true"
-        else:
-            addurl=""
-
-        metaurl = self.url+addurl
-        url = self.url+'/navigate'+addurl
-        logger.info("url: "+url)
-        logger.info("metaurl: "+metaurl)
-
-        try:
-            data = self._fetchUrl(url)
-            meta = self._fetchUrl(metaurl)
-
-            if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
-            else:
-                raise e
-                
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url,data)
-            data = self._fetchUrl(url)
-            meta = self._fetchUrl(metaurl)
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        metasoup = bs.BeautifulSoup(meta)
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
-        self.story.setMetadata('title',a.string)
-		
-        # Find authorid and URL from... author url.
-        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
-        if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
-            self.story.setMetadata('author','Anonymous')
-            self.story.setMetadata('authorUrl','http://archiveofourown.org/')
-            self.story.setMetadata('authorId','0')
-        else:
-            for a in alist:
-                self.story.addToList('authorId',a['href'].split('/')[2])
-                self.story.addToList('authorUrl','http://'+self.host+a['href'])
-                self.story.addToList('author',a.text)
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
-        self.story.setMetadata('numChapters',len(chapters))
-        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
-            else:
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-
-
-
-        a = metasoup.find('blockquote',{'class':'userstuff'})
-        if a != None:
-            self.setDescription(url,a)
-            #self.story.setMetadata('description',a.text)
-		
-        a = metasoup.find('dd',{'class':"rating tags"})
-        if a != None:
-            self.story.setMetadata('rating',stripHTML(a.text))
-		
-        a = metasoup.find('dd',{'class':"fandom tags"})
-        fandoms = a.findAll('a',{'class':"tag"})
-        for fandom in fandoms:
-            self.story.addToList('fandoms',fandom.string)
-            self.story.addToList('category',fandom.string)
-		
-        a = metasoup.find('dd',{'class':"warning tags"})
-        if a != None:
-            warnings = a.findAll('a',{'class':"tag"})
-            for warning in warnings:
-                if warning.string == "Author Chose Not To Use Archive Warnings":
-                    warning.string = "No Archive Warnings Apply"
-                if warning.string != "No Archive Warnings Apply":
-                    self.story.addToList('warnings',warning.string)
-		
-        a = metasoup.find('dd',{'class':"freeform tags"})
-        if a != None:
-            genres = a.findAll('a',{'class':"tag"})
-            for genre in genres:
-                self.story.addToList('freefromtags',genre.string)
-                self.story.addToList('genre',genre.string)
-                
-        a = metasoup.find('dd',{'class':"category tags"})
-        if a != None:
-            genres = a.findAll('a',{'class':"tag"})
-            for genre in genres:
-                if genre != "Gen":
-                    self.story.addToList('ao3categories',genre.string)
-                    self.story.addToList('genre',genre.string)
-		
-        a = metasoup.find('dd',{'class':"character tags"})
-        if a != None:
-            chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
-                self.story.addToList('characters',char.string)
-        a = metasoup.find('dd',{'class':"relationship tags"})
-        if a != None:
-            ships = a.findAll('a',{'class':"tag"})
-            for ship in ships:
-                self.story.addToList('ships',ship.string)
-		
-
-        stats = metasoup.find('dl',{'class':'stats'})
-        dt = stats.findAll('dt')
-        dd = stats.findAll('dd')
-        for x in range(0,len(dt)):
-            label = dt[x].text
-            value = dd[x].text
-
-            if 'Words:' in label:
-                self.story.setMetadata('numWords', value)
-				
-            if 'Comments:' in label:
-                self.story.setMetadata('comments', value)
-				
-            if 'Kudos:' in label:
-                self.story.setMetadata('kudos', value)
-				
-            if 'Hits:' in label:
-                self.story.setMetadata('hits', value)
-				
-            if 'Bookmarks:' in label:
-                self.story.setMetadata('bookmarks', value)
-				
-            if 'Chapters:' in label:
-                if value.split('/')[0] == value.split('/')[1]:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-				
-            if 'Completed' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-		
-        try:
-            # Find Series name from series URL.
-            a = metasoup.find('dd',{'class':"series"})
-            b = a.find('a', href=re.compile(r"/series/\d+"))
-            series_name = b.string
-            series_url = 'http://'+self.host+'/fanfic/'+b['href']
-            series_index = int(a.text.split(' ')[1])
-            self.setSeries(series_name, series_index)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
-        data = self._fetchUrl(url)
-        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
-            if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
-        
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
-            if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
-		
-        text = soup.find('div', {'class' : "userstuff module"})
-        chtext = text.find('h3', {'class' : "landmark heading"})
-        if chtext:
-            chtext.extract()
-        chapter.append(text)
-		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
-		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
-			
-        if None == soup:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,chapter)

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -1,191 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return ArchiveSkyeHawkeComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ash')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y-%m-%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'archive.skyehawke.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
-
-
-        
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        data = self._fetchUrl(url)
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=a.text.split('"')[1]
-        self.story.setMetadata('title',title)
-        
-        # Find authorid and URL from... author url.
-        author = a.find('a')
-        self.story.setMetadata('authorId',author['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+author['href'])
-        self.story.setMetadata('author',author.string)
-		
-        authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        chapter=soup.find('select',{'name':'chapter'}).findAll('option')
-	
-        for i in range(1,len(chapter)):
-            ch=chapter[i]
-            self.chapterUrls.append((stripHTML(ch),ch['value']))
-		
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-		
-        box=soup.find('div', {'class': "container borderridge"})
-        sum=box.find('span').text
-        self.setDescription(url,sum)
-
-        boxes=soup.findAll('div', {'class': "container bordersolid"})
-        for box in boxes:
-            if box.find('b') != None and box.find('b').text == "History and Story Information":
-			
-                for b in box.findAll('b'):
-                    if "words" in b.nextSibling:
-                        self.story.setMetadata('numWords', b.text)
-                    if "archived" in b.previousSibling:
-                        self.story.setMetadata('datePublished', makeDate(stripHTML(b.text), self.dateformat))
-                    if "updated" in b.previousSibling:
-                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(b.text), self.dateformat))
-                    if "fandom" in b.nextSibling:
-                        self.story.addToList('category', b.text)
-						
-                for br in box.findAll('br'):
-                    br.replaceWith('split')
-                genre=box.text.split("Genre:")[1].split("split")[0]
-                if not "Unspecified" in genre:
-                    self.story.addToList('genre',genre)
-			
-			
-            if box.find('span') != None and box.find('span').text == "WARNING":
-			
-                rating=box.findAll('span')[1]
-                rating.find('br').replaceWith('split')
-                rating=rating.text.replace("This story is rated",'').split('split')[0]
-                self.story.setMetadata('rating',rating)
-                logger.debug(self.story.getMetadata('rating'))
-			
-                warnings=box.find('ol')
-                if warnings != None:
-                    warnings=warnings.text.replace(']', '').replace('[', '').split('  ')
-                    for warning in warnings:
-                        self.story.addToList('warnings',warning)
-	
-	
-        for asoup in authorSoup.findAll('div', {'class':"story bordersolid"}):
-            if asoup.find('a')['href'] == 'story.php?no='+self.story.getMetadata('storyId'):
-                if '[ Completed ]' in asoup.text:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-                chars=asoup.findNext('div').text.split('Characters')[1].split(']')[0]
-                for char in chars.split(','):
-                    if not "None" in char:
-                        self.story.addToList('characters',char)
-                break
-	
-	
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div',{'class':"chapter bordersolid"}).findNext('div').findNext('div')
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -1,252 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return AshwinderSycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','asph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'ashwinder.sycophanthex.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'This story contains adult content and/or themes.' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Logout" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                self.story.setMetadata('title',a.string)
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'table':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = info.findAll('b')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word Count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if genre != 'none':
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if warning != ' none':
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"align" : "left"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -1,334 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# By virtue of being recent and requiring both is_adult and user/pass,
-# adapter_fanficcastletvnet.py is the best choice for learning to
-# write adapters--especially for sites that use the eFiction system.
-# Most sites that have ".../viewstory.php?sid=123" in the story URL
-# are eFiction.
-
-# For non-eFiction sites, it can be considerably more complex, but
-# this is still a good starting point.
-
-# In general an 'adapter' needs to do these five things:
-
-# - 'Register' correctly with the downloader
-# - Site Login (if needed)
-# - 'Are you adult?' check (if needed--some do one, some the other, some both)
-# - Grab the chapter list
-# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
-# - Grab the chapter texts
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return BloodTiesFansComAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','btf') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'bloodties-fans.com' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-
-            # Furthermore, there's a couple sites now with more than
-            # one warning level for different ratings.  And they're
-            # fussy about it.  midnightwhispers has three: 4, 2 & 1.
-            # we'll try 1 first.
-            addurl = "&ageconsent=ok&warning=4" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  nfacommunity uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=561&warning=4
-        # viewstory.php?sid=561&warning=1
-        # viewstory.php?sid=561&warning=2
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/fiction/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -1,307 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# By virtue of being recent and requiring both is_adult and user/pass,
-# adapter_fanficcastletvnet.py is the best choice for learning to
-# write adapters--especially for sites that use the eFiction system.
-# Most sites that have ".../viewstory.php?sid=123" in the story URL
-# are eFiction.
-
-# For non-eFiction sites, it can be considerably more complex, but
-# this is still a good starting point.
-
-# In general an 'adapter' needs to do these five things:
-
-# - 'Register' correctly with the downloader
-# - Site Login (if needed)
-# - 'Are you adult?' check (if needed--some do one, some the other, some both)
-# - Grab the chapter list
-# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
-# - Grab the chapter texts
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return CastleFansOrgAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/fanfic/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','cslf') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'castlefans.org' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Age Consent Required" in data: # XXX 
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        pagetitle = soup.find('div',{'id':'pagetitle'})
-        ## Title
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfic/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/fanfic/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -1,235 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ChaosSycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ChaosSycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','csph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'chaos.sycophanthex.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=19"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Age Consent Required" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
-        
-        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        rating=pt.text.split('(')[1].split(')')[0]
-        self.story.setMetadata('rating', rating)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-				
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-		
-        labels = soup.findAll('span',{'class':'label'})
-		
-        value = labels[0].previousSibling
-        svalue = ""
-        while value != None:
-            val = value
-            value = value.previousSibling
-        while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-		
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value.split(' -')[0])
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Complete' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -1,237 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return CheckmatedComAdapter
-
-
-class CheckmatedComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','chm')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.checkmated.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?story=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?story=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites. This story is in The Bedchamber
-    def needToLoginCheck(self, data):
-        if 'This story is in The Bedchamber' in data \
-                or 'That username is not in our database' in data \
-                or "That password is not correct, please try again" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['name'] = self.username
-            params['pass'] = self.password
-        else:
-            params['name'] = self.getConfig("username")
-            params['pass'] = self.getConfig("password")
-        params['login'] = 'yes'
-        params['submit'] = 'login'
-
-        loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
-        d = self._fetchUrl(loginUrl,params)
-        e = self._fetchUrl(url)
-
-        if "Welcome back," not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['name']))
-            raise exceptions.FailedToLogin(url,params['name'])
-            return False
-        elif "This story is in The Bedchamber" in e:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Your account does not have sufficient priviliges to read this story.")
-            return False
-        else:
-            return True
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        ## Title
-        a = soup.findAll('span', {'class' : 'storytitle'})
-        self.story.setMetadata('title',a[0].string)
-
-        # Find authorid and URL from... author url.
-        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        a = soup.find('select', {'name' : 'chapter'})
-        if a == None:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in a.findAll('option'):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-
-        # website does not keep track of word count, and there is no convenient way to calculate it
-
-        summary = soup.find('fieldset')
-        summary.find('legend').extract()
-        summary.name='div'
-        self.setDescription(url,summary)
-
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        table = soup.findAll('div', {'class' : 'text'})[1]
-        for labels in table.findAll('tr'):
-            value = labels.findAll('td')[1]
-            label = labels.findAll('td')[0]
-
-
-            if 'Rating' in stripHTML(label):
-                self.story.setMetadata('rating', stripHTML(value))
-
-            if 'Ship' in stripHTML(label):
-                if value.string != "none/none":
-                    self.story.addToList('ships',value.string)
-                for char in value.string.split('/'):
-                    if char != 'none':
-                        self.story.addToList('characters',char)
-
-            if 'Status' in stripHTML(label):
-                if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in stripHTML(label):
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in stripHTML(label):
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')
-        for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
-            a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
-            if a != None:
-                for labels in story.findAll('tr'):
-                    value = labels.findAll('td')[1]
-                    label = labels.findAll('td')[0]
-                    if 'genre' in stripHTML(label):
-                        for genre in value.findAll('img'):
-                            self.story.addToList('genre',genre['title'])
-
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        div = soup.find('div', {'id' : 'resizeableText'})
-        div.find('div', {'class' : 'storyTools'}).extract()
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -1,275 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DarkSolaceOrgAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DarkSolaceOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dksl')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'dark-solace.org'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.dark-solace.org','dark-solace.org']
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "User Account Page" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            addurl="&ageconsent=ok"
-            self.performLogin(url)
-            data = self._fetchUrl(url+addurl)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select', {'name' : 'chapter'})
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
-            if a != None:
-                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
-                    break
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
-        for labelspan in labels:
-            label = labelspan.text
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
-
-            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/elysian/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -1,241 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DestinysGatewayComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DestinysGatewayComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dgrfa')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.destinysgateway.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dokugacom.py

@@ -1,238 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DokugaComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DokugaComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        # www.dokuga.com has two 'sections', shown in URL as
-        # 'fanfiction' and 'spark' that change how things should be
-        # handled.
-        # http://www.dokuga.com/fanfiction/story/7528/1
-        # http://www.dokuga.com/spark/story/7299/1
-        self.section=self.parsedUrl.path.split('/',)[1]
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dkg')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        if 'fanfiction' in self.section:
-            self.dateformat = "%d %b %Y"
-        else:
-            self.dateformat = "%m-%d-%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.dokuga.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
-
-    def getSiteURLPattern(self):
-        return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'align' : 'center'}).find('h3')
-        
-        # Find authorid and URL from... author url.
-        aut = a.find('a')
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        alink='http://'+self.host+aut['href']
-        self.story.setMetadata('authorUrl','http://'+self.host+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        a = a.string[:(len(a.string)-4)]
-        self.story.setMetadata('title',a)
-
-        # Find the chapters:
-        chapters = soup.find('select').findAll('option')
-        if len(chapters)==1:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1'))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(alink))
-        
-        if 'fanfiction' in self.section:
-            asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
-        
-            #grab the rest of the metadata from the author's page
-            for div in asoup.findAll('div'):
-                nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
-                if nav != None:
-                    break
-            div=div.nextSibling
-            self.setDescription(url,div)
-        
-            div=div.nextSibling
-            
-            a=div.text.split('Rating: ')
-            if len(a) == 2: self.story.setMetadata('rating', a[1].split('&')[0])
-        
-            a=div.text.split('Status: ')
-            if len(a)==2:
-                iscomp=a[1].split('&')[0]
-                if 'Complete' in iscomp:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-            
-            a=div.text.split('Category: ')
-            if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
-            self.story.addToList('category', 'Fanfiction')
-        
-            a=div.text.split('Created: ')
-            if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
-        
-            a=div.text.split('Updated: ')
-            if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1]), self.dateformat))
-            
-            div=div.nextSibling.nextSibling
-            a=div.text.split('Words: ')
-            if len(a) == 2: self.story.setMetadata('numWords', a[1].split('&')[0])
-        
-            a=div.text.split('Genre: ')
-            if len(a) == 2:
-                for genre in a[1].split('&')[0].split(', '):
-                    self.story.addToList('genre',genre)
-        
-        else:
-            asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
-            for div in asoup.findAll('div'):
-                nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
-                if nav != None:
-                    break
-                    
-            div=div.nextSibling.nextSibling
-            self.setDescription(url,div)
-            self.story.addToList('category', 'Spark')
-            
-            div=div.nextSibling.nextSibling
-            a=div.text.split('Rating: ')
-            if len(a) == 2: self.story.setMetadata('rating', a[1].split(' - ')[0])
-            
-            a=div.text.split('Status: ')
-            if len(a)==2:
-                iscomp=a[1].split(' - ')[0]
-                if 'Complete' in iscomp:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-            
-            a=div.text.split('Genre: ')
-            if len(a)==2:
-                for genre in a[1].split(' - ')[0].split('/'):
-                    self.story.addToList('genre',genre)
-                
-            div=div.nextSibling.nextSibling
-            
-            a=div.text.split('Updated: ')
-            if len(a)==2:
-                date=a[1].split('      -')[0]
-                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
-        
-                # does not have published date anywhere
-                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-            
-            a=div.text.split('Words ')
-            if len(a)==2: self.story.setMetadata('numWords', a[1])
-            
-        
-
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'chtext'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -1,299 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DracoAndGinnyComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DracoAndGinnyComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dcagn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.dracoandginny.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=2"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        content=soup.find('div',{'class' : 'listbox'})
-        
-        self.setDescription(url,content.find('blockquote'))
-        
-        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
-            self.story.addToList('genre',genre.string)
-
-        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
-            self.story.addToList('warnings',warning.string)
-        
-        labels = content.findAll('b')
-        
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value.split(' |')[0])
-                
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.nextSibling).split(' |')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'class' : 'listbox'})
-        
-        if None == div:
-            div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -1,297 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return DramioneOrgAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DramioneOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','drmn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'dramione.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Stories that are suitable for ages 16 and older" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Use banner as cover if found
-        coverurl = ''
-        img = soup.find('img',{'class':'banner'})
-        if img:
-            coverurl = img['src']
-            #print "Cover: "+coverurl
-        a = soup.find(text="This story has a banner; click to view.")
-        if a:
-            #print "A: "+ ', '.join("(%s, %s)" %tup for tup in a.parent.attrs)
-            coverurl = a.parent['href']
-            #print "Cover: "+coverurl
-        if coverurl:
-            self.setCoverImage(url,coverurl)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-		
-        genres=soup.findAll('a', {'class' : "tag-1"})
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-
-        warnings=soup.findAll('a', {'class' : "tag-2"})
-        for warning in warnings:
-            self.story.addToList('warnings',warning.string)
-
-        themes=soup.findAll('a', {'class' : "tag-3"})
-        for theme in themes:
-            self.story.addToList('themes',theme.string)
-
-        hermiones=soup.findAll('a', {'class' : "tag-4"})
-        for hermione in hermiones:
-            self.story.addToList('hermiones',hermione.string)
-
-        dracos=soup.findAll('a', {'class' : "tag-5"})
-        for draco in dracos:
-            self.story.addToList('dracos',draco.string)
-
-        timelines=soup.findAll('a', {'class' : "tag-6"})
-        for timeline in timelines:
-            self.story.addToList('timeline',timeline.string)
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -1,253 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ErosnSapphoSycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','essph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'erosnsappho.sycophanthex.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=18"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
-        
-        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        rating=pt.text.split('(')[1].split(')')[0]
-        self.story.setMetadata('rating', rating)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-				
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-		
-        labels = soup.findAll('span',{'class':'label'})
-		
-        value = labels[0].previousSibling
-        svalue = ""
-        while value != None:
-            val = value
-            value = value.previousSibling
-        while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-		
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value.split(' -')[0])
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Complete' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -1,276 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-from urllib import unquote_plus
-import time
-
-from .. import BeautifulSoup as bs
-from .. import exceptions as exceptions
-from ..htmlcleanup import stripHTML
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-ffnetgenres=["Adventure", "Angst", "Crime", "Drama", "Family", "Fantasy", "Friendship", "General",
-             "Horror", "Humor", "Hurt-Comfort", "Mystery", "Parody", "Poetry", "Romance", "Sci-Fi",
-             "Spiritual", "Supernatural", "Suspense", "Tragedy", "Western"]
-
-class FanFictionNetSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','ffnet')
-        
-        # get storyId from url--url validation guarantees second part is storyId
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-
-        # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
-                         +"/s/"+self.story.getMetadata('storyId')+"/1/")
-
-        # ffnet update emails have the latest chapter URL.
-        # Frequently, when they arrive, not all the servers have the
-        # latest chapter yet and going back to chapter 1 to pull the
-        # chapter list doesn't get the latest.  So save and use the
-        # original URL given to pull chapter list & metadata.
-        self.origurl = url
-        if "http://m." in self.origurl:
-            ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("http://m.","http://www.")
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.fanfiction.net'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.fanfiction.net','m.fanfiction.net']
-
-    def getSiteExampleURLs(self):
-        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
-
-    def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        # fetch the chapter.  From that we will get almost all the
-        # metadata and chapter list
-
-        url = self.origurl
-        logger.debug("URL: "+url)
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        try:
-            data = self._fetchUrl(url)
-            #logger.debug("\n===================\n%s\n===================\n"%data)
-            soup = bs.BeautifulSoup(data)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(url)
-            else:
-                raise e
-            
-        if "Unable to locate story" in data:
-            raise exceptions.StoryDoesNotExist(url)
-
-        # some times "Chapter not found...", sometimes "Chapter text not found..."
-        if "not found. Please check to see you are not using an outdated url." in data:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
-
-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
-            try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
-            except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                           self.story.getMetadata('storyId'),
-                                           chapcount+1)
-            logger.debug('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                logger.debug('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/u/\d+"))
-        self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        ## Pull some additional data from html.
-
-        ## ffnet shows category two ways
-        ## 1) class(Book, TV, Game,etc) >> category(Harry Potter, Sailor Moon, etc)
-        ## 2) cat1_cat2_Crossover
-        ## For 1, use the second link.
-        ## For 2, fetch the crossover page and pull the two categories from there.
-
-        categories = soup.findAll('a',{'class':'xcontrast_txt'})
-        if len(categories) > 1:
-            self.story.addToList('category',stripHTML(categories[-1]))
-        elif 'Crossover' in categories[0]['href']:
-            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
-            catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
-            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
-                self.story.addToList('category',stripHTML(a))
-            
-        a = soup.find('a', href='http://www.fictionratings.com/')
-        rating = a.string
-        if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
-            rating = rating[8:]
-            
-        self.story.setMetadata('rating',rating)
-
-        # after Rating, the same bit of text containing id:123456 contains
-        # Complete--if completed.
-        gui_table1i = soup.find(id="gui_table1i")
-
-        self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
-
-        summarydiv = gui_table1i.find('div',{'style':'margin-top:2px'})
-        if summarydiv:
-            self.setDescription(url,stripHTML(summarydiv))
-            
-        
-        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
-        metalist = metatext.split(" - ")
-        #logger.debug("metatext:(%s)"%metalist)
-
-        # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
-
-        # rating is obtained above more robustly.
-        if metalist[0].startswith('Rated:'):
-            metalist=metalist[1:]
-
-        # next is assumed to be language.
-        self.story.setMetadata('language',metalist[0])
-        metalist=metalist[1:]
-
-        # next might be genre.
-        genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
-        goodgenres=True
-        for g in genrelist:
-            #logger.debug("g:(%s)"%g)
-            if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
-                goodgenres=False
-        if goodgenres:
-            self.story.extendList('genre',genrelist)
-            metalist=metalist[1:]
-
-        donechars = False
-        while len(metalist) > 0:
-            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
-                pass
-            elif  metalist[0].startswith('Reviews'):
-                self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Favs:'):
-                self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Follows:'):
-                self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Updated'):
-                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
-            elif  metalist[0].startswith('Published'):
-                self.story.setMetadata('datePublished',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
-            elif  metalist[0].startswith('Words'):
-                self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
-            elif not donechars:
-                self.story.extendList('characters',metalist[0].split('&'))
-                donechars = True
-            metalist=metalist[1:]
-
-        # next might be characters, otherwise Reviews, Updated, Published, Words
-        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
-        #     self.story.extendList('characters',metalist[0].split('&'))
-        
-        if 'Status: Complete' in metatext:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        img = soup.find('img',{'class':'cimage'})
-        if img:
-            self.setCoverImage(url,img['src'])
-            
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapter' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = u'http://%s/s/%s/%s/' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            o['value'])
-                # just in case there's tags, like <i> in chapter titles.
-                title = u"%s" % o
-                title = re.sub(r'<[^>]+>','',title)
-                self.chapterUrls.append((title,url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        return
-
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
-                        ## more if hit too fast.  This is in
-                        ## additional to what ever the
-                        ## slow_down_sleep_time setting is.
-        data = self._fetchUrl(url)
-
-        # some ancient stories have body tags inside them that cause
-        # soup parsing to discard the content.  For story text we
-        # don't care about anything before "<div class='storytextp"
-        # (there's a space after storytextp, so no close quote(')) and
-        # this kills any body tags.
-        data = data[data.index("<div class='storytextp"):]
-        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
-        
-        soup = bs.BeautifulSoup(data)
-
-        ## Remove the 'share' button.
-        sharediv = soup.find('div', {'class' : 'a2a_kit a2a_default_style'})
-        if sharediv:
-            sharediv.extract()
-        
-        div = soup.find('div', {'id' : 'storytextp'})
-        
-        if None == div:
-            logger.debug('div id=storytextp not found.  data:%s'%data)
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)
-
-def getClass():
-    return FanFictionNetSiteAdapter
-

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -1,200 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return FanFiktionDeAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class FanFiktionDeAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ffde')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d.%m.%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.fanfiktion.de'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
-        
-        ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Diese Geschichte wurde als entwicklungsbeeintr' in data \
-                or 'There is no such account on our website' in data \
-                or "Noch kein registrierter Benutzer?" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self,url):
-        params = {}
-
-        if self.password:
-            params['nickname'] = self.username
-            params['passwd'] = self.password
-        else:
-            params['nickname'] = self.getConfig("username")
-            params['passwd'] = self.getConfig("password")
-        params['savelogindata'] = '1'
-        params['a'] = 'l'
-        params['submit'] = 'Login...'
-
-        loginUrl = 'https://ssl.fanfiktion.de/'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['nickname']))
-        d = self._postUrl(loginUrl,params)
-    
-        if "Login erfolgreich" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['nickname']))
-            raise exceptions.FailedToLogin(url,params['nickname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-                
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Uhr ist diese Geschichte nur nach einer" in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Auserhalb der Zeit von 23:00 Uhr bis 04:00 Uhr ist diese Geschichte nur nach einer erfolgreichen Altersverifikation zuganglich.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        head = soup.find('div', {'style' : 'width:85%;float:left;'})
-        a = head.find('a')
-        self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.find('select').findAll('option'):
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        self.story.setMetadata('language','German')
-
-        #find metadata on the story page
-        self.story.setMetadata('datePublished', makeDate(head.text.split('erstellt: ')[1].split('\n')[0], self.dateformat))
-        
-        self.story.setMetadata('dateUpdated', makeDate(head.text.split('letztes Update: ')[1].split('\n')[0], self.dateformat))
-        
-        for genre in head.text.split('   ')[3].split('/')[0].split(', '):
-            self.story.addToList('genre',genre)
-            
-        if 'fertiggestellt' in head.text:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In Progress')
-
-        #find metadata on the author's page
-        asoup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
-        tr=asoup.findAll('tr')
-        for i in range(1,len(tr)):
-            a = tr[i].find('a')
-            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
-                break
-        self.setDescription(url,a['onmouseover'].split("', '")[1])
-        
-        td = tr[i].findAll('td')
-        self.story.addToList('category',td[1].string)
-        self.story.setMetadata('rating', td[4].string)
-        self.story.setMetadata('numWords', td[5].string)
- 
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffde has "floodlock" protection
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'storytext'})
-        for a in div.findAll('script'):
-            a.extract()
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -1,225 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import datetime
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-from .. import translit
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return FicBookNetAdapter
-
-
-logger = logging.getLogger(__name__)
-
-class FicBookNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','fbn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %m %Y" 
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.ficbook.net'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/readfic/12345"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url=self.url
-        logger.debug("URL: "+url)
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-				
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        # Now go hunting for all the meta data and the chapter list.
-		
-        table = soup.find('td',{'width':'50%'})
-        
-        ## Title
-        a = soup.find('h1')
-        self.story.setMetadata('title',stripHTML(a))
-        logger.debug("Title: (%s)"%self.story.getMetadata('title'))
-        
-        # Find authorid and URL from... author url.
-        a = table.find('a')
-        self.story.setMetadata('authorId',a.text) # Author's name is unique
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.text)
-        logger.debug("Author: (%s)"%self.story.getMetadata('author'))
-
-        # Find the chapters:
-        chapters = soup.find('div', {'class' : 'part_list'})
-        if chapters != None:
-            chapters=chapters.findAll('a', href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+"/\d+#part_content$"))
-            self.story.setMetadata('numChapters',len(chapters))
-            for x in range(0,len(chapters)):
-                chapter=chapters[x]
-                churl='http://'+self.host+chapter['href']
-                self.chapterUrls.append((stripHTML(chapter),churl))
-                if x == 0:
-                    pubdate = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
-                if x == len(chapters)-1:
-                    update = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-            self.story.setMetadata('numChapters',1)
-            pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
-            update=pubdate
-
-        logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-
-        if not ',' in pubdate:
-            pubdate=datetime.date.today().strftime(self.dateformat)
-        if not ',' in update:
-            update=datetime.date.today().strftime(self.dateformat)
-        pubdate=pubdate.split(',')[0]
-        update=update.split(',')[0]
-		
-        fullmon = {"yanvarya":"01", "января":"01",
-           "fievralya":"02", "февраля":"02",
-           "marta":"03", "марта":"03",
-           "aprielya":"04", "апреля":"04",
-           "maya":"05", "мая":"05",
-           "iyunya":"06", "июня":"06",
-           "iyulya":"07", "июля":"07",
-           "avghusta":"08", "августа":"08",
-           "sentyabrya":"09", "сентября":"09",
-           "oktyabrya":"10", "октября":"10",
-           "noyabrya":"11", "ноября":"11",
-           "diekabrya":"12", "декабря":"12" }
-        for (name,num) in fullmon.items():
-            if name in pubdate:
-                pubdate = pubdate.replace(name,num)
-            if name in update:
-                update = update.replace(name,num)
-
-        self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
-        self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
-        self.story.setMetadata('language','Russian')
-		
-        pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
-        pr='http://'+self.host+pr['href']
-        pr = bs.BeautifulSoup(self._fetchUrl(pr))
-        pr=pr.findAll('div', {'class' : 'part_text'})
-        i=0
-        for part in pr:
-            i=i+len(stripHTML(part).split(' '))
-        self.story.setMetadata('numWords', str(i))
-		
-        i=0
-        fandoms = table.findAll('a', href=re.compile(r'/fanfiction/\w+'))
-        for fandom in fandoms:
-            self.story.addToList('category',fandom.string)
-            i=i+1
-        if i > 1:
-            self.story.addToList('genre', 'Кроссовер')
-		
-        meta=table.findAll('a', href=re.compile(r'/ratings/'))
-        i=0
-        for m in meta:
-            if i == 0:
-                self.story.setMetadata('rating', m.find('b').text)
-                i=1
-            elif i == 1:
-                if not "," in m.nextSibling:
-                    i=2
-                self.story.addToList('genre', m.find('b').text)
-            elif i == 2:
-                self.story.addToList('warnings', m.find('b').text)
-		
-
-        if table.find('span', {'style' : 'color: green'}):
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In Progress')
-		
-
-        tags = table.findAll('b')
-        for tag in tags:
-            label = translit.translit(tag.text)
-            if 'Piersonazhi:' in label or 'Персонажи:' in label:
-                chars=tag.nextSibling.string.split(', ')
-                for char in chars:
-                    self.story.addToList('characters',char)
-                break
-				
-        summary=soup.find('span', {'class' : 'urlize'})
-        self.setDescription(url,summary)
-        #self.story.setMetadata('description', summary.text)
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        chapter = soup.find('div', {'class' : 'public_beta'})
-        if chapter == None:
-            chapter = soup.find('div', {'class' : 'public_beta_disabled'})
-
-        if None == chapter:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,chapter)

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -1,231 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','fa')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('authorId',m.group('auth'))
-            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-            # normalized story URL.
-            self._setURL(url)
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'www.fictionalley.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/authors/drt/DA.html http://"+self.getSiteDomain()+"/authors/drt/JOTP01a.html"
-
-    def getSiteURLPattern(self):
-        # http://www.fictionalley.org/authors/drt/DA.html
-        # http://www.fictionalley.org/authors/drt/JOTP01a.html
-        return re.escape("http://"+self.getSiteDomain())+"/authors/(?P<auth>[a-zA-Z0-9_]+)/(?P<id>[a-zA-Z0-9_]+)\.html"
-
-    def _postFetchWithIAmOld(self,url):
-        if self.is_adult or self.getConfig("is_adult"):
-            params={'iamold':'Yes',
-                    'action':'ageanswer'}
-            logger.info("Attempting to get cookie for %s" % url)
-            ## posting on list doesn't work, but doesn't hurt, either.
-            data = self._postUrl(url,params)
-        else:
-            data = self._fetchUrl(url)
-        return data
-
-    def extractChapterUrlsAndMetadata(self):
-
-        ## could be either chapter list page or one-shot text page.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._postFetchWithIAmOld(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        chapterdata = data
-        # If chapter list page, get the first chapter to look for adult check
-        chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
-        if chapterlinklist:
-            chapterdata = self._postFetchWithIAmOld(chapterlinklist[0]['href'])
-            
-        if "Are you over seventeen years old" in chapterdata:
-            raise exceptions.AdultCheckRequired(self.url)
-        
-        if not chapterlinklist:
-            # no chapter list, chapter URL: change to list link.
-            # second a tag inside div breadcrumbs
-            storya = soup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self._setURL(storya['href'])
-            url=self.url
-            logger.debug("Normalizing to URL: "+url)
-            ## title's right there...
-            self.story.setMetadata('title',storya.string)
-            data = self._fetchUrl(url)
-            soup = bs.BeautifulSoup(data)
-            chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
-        else:
-            ## still need title from somewhere.  If chapterlinklist,
-            ## then chapterdata contains a chapter, find title the
-            ## same way.
-            chapsoup = bs.BeautifulSoup(chapterdata)
-            storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self.story.setMetadata('title',storya.string)
-            del chapsoup
-
-        del chapterdata
-        
-        ## authorid already set.
-        ## <h1 class="title" align="center">Just Off The Platform II by <a href="http://www.fictionalley.org/authors/drt/">DrT</a></h1>
-        authora=soup.find('h1',{'class':'title'}).find('a')
-        self.story.setMetadata('author',authora.string)
-        self.story.setMetadata('authorUrl',authora['href'])
-
-        if len(chapterlinklist) == 1:
-            self.chapterUrls.append((self.story.getMetadata('title'),chapterlinklist[0]['href']))
-        else:
-            # Find the chapters:
-            for chapter in chapterlinklist:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        ## Go scrape the rest of the metadata from the author's page.
-        data = self._fetchUrl(self.story.getMetadata('authorUrl'))
-        soup = bs.BeautifulSoup(data)
-
-        # <dl><dt><a class = "Rid story" href = "http://www.fictionalley.org/authors/aafro_man_ziegod/TMH.html">
-        # [Rid] The Magical Hottiez</a> by <a class = "pen_name" href = "http://www.fictionalley.org/authors/aafro_man_ziegod/">Aafro Man Ziegod</a> </small></dt>
-        # <dd><small class = "storyinfo"><a href = "http://www.fictionalley.org/ratings.html" target = "_new">Rating:</a> PG-13 - Spoilers: PS/SS, CoS, PoA, GoF, QTTA, FB - 4264 hits - 5060 words<br />
-        # Genre: Humor, Romance - Main character(s): None - Ships: None - Era: Multiple Eras<br /></small>
-        # Chaos ensues after Witch Weekly, seeking to increase readers, decides to create a boyband out of five seemingly talentless wizards: Harry Potter, Draco Malfoy, Ron Weasley, Neville Longbottom, and Oliver "Toss Your Knickers Here" Wood.<br />
-        # <small class = "storyinfo">Published: June 3, 2002 (between Goblet of Fire and Order of Phoenix) - Updated: June 3, 2002</small>
-        # </dd></dl>
-        
-        storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
-        storydd = storya.findNext('dd')
-
-        # Rating: PG - Spoilers: None - 2525 hits - 736 words
-        # Genre: Humor - Main character(s): H, R - Ships: None - Era: Multiple Eras
-        # Harry and Ron are back at it again! They reeeeeeally don't want to be back, because they know what's awaiting them. "VH1 Goes Inside..." is back! Why? 'Cos there are soooo many more couples left to pick on.
-        # Published: September 25, 2004 (between Order of Phoenix and Half-Blood Prince) - Updated: September 25, 2004 
-               
-        ## change to text and regexp find.
-        metastr = stripHTML(storydd).replace('\n',' ').replace('\t',' ')
-
-        m = re.match(r".*?Rating: (.+?) -.*?",metastr)
-        if m:
-            self.story.setMetadata('rating', m.group(1))
-
-        m = re.match(r".*?Genre: (.+?) -.*?",metastr)
-        if m:
-            for g in m.group(1).split(','):
-                self.story.addToList('genre',g)
-        
-        m = re.match(r".*?Published: ([a-zA-Z]+ \d\d?, \d\d\d\d).*?",metastr)
-        if m:
-            self.story.setMetadata('datePublished',makeDate(m.group(1), "%B %d, %Y"))
-
-        m = re.match(r".*?Updated: ([a-zA-Z]+ \d\d?, \d\d\d\d).*?",metastr)
-        if m:
-            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%B %d, %Y"))
-
-        m = re.match(r".*? (\d+) words Genre.*?",metastr)
-        if m:
-            self.story.setMetadata('numWords', m.group(1))
-            
-        for small in storydd.findAll('small'):
-            small.extract() ## removes the <small> tags, leaving only the summary.
-        self.setDescription(url,storydd)
-        #self.story.setMetadata('description',stripHTML(storydd))
-        
-        return
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-	# find <!-- headerend --> & <!-- footerstart --> and
-	# replaced with matching div pair for easier parsing.
-	# Yes, it's an evil kludge, but what can ya do?  Using
-	# something other than div prevents soup from pairing
-	# our div with poor html inside the story text.
-	data = data.replace('<!-- headerend -->','<crazytagstringnobodywouldstumbleonaccidently id="storytext">').replace('<!-- footerstart -->','</crazytagstringnobodywouldstumbleonaccidently>')
-        
-        # problems with some stories confusing Soup.  This is a nasty
-        # hack, but it works.
-        data = data[data.index("<crazytagstringnobodywouldstumbleonaccidently"):]
-
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        body = soup.findAll('body') ## some stories use a nested body and body
-                                    ## tag, in which case we don't
-                                    ## need crazytagstringnobodywouldstumbleonaccidently
-                                    ## and use the second one instead.
-        if len(body)>1:
-            text = body[1]
-            text.name='div' # force to be a div to avoid multiple body tags.
-        else:
-            text = soup.find('crazytagstringnobodywouldstumbleonaccidently', {'id' : 'storytext'})
-            text.name='div' # change to div tag.
-            
-        if not data or not text:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,text)
-
-def getClass():
-    return FictionAlleyOrgSiteAdapter
-

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import time
-
-## They're from the same people and pretty much identical.
-from adapter_fanfictionnet import FanFictionNetSiteAdapter
-
-class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
-
-    def __init__(self, config, url):
-        FanFictionNetSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','fpcom')
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.fictionpress.com'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.fictionpress.com','m.fictionpress.com']
-
-    def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title"
-
-    def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
-
-def getClass():
-    return FictionPressComSiteAdapter
-

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -1,218 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import time
-import httplib, urllib
-
-from .. import BeautifulSoup as bs
-from .. import exceptions as exceptions
-from ..htmlcleanup import stripHTML
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class FicwadComSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','fw')
-        
-        # get storyId from url--url validation guarantees second part is storyId
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        
-        self.username = "NoneGiven"
-        self.password = ""
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.ficwad.com'
-
-    def getSiteExampleURLs(self):
-        return "http://www.ficwad.com/story/137169"
-
-    def getSiteURLPattern(self):
-        return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"
-
-    def performLogin(self,url):
-        params = {}
-
-        if self.password:
-            params['username'] = self.username
-            params['password'] = self.password
-        else:
-            params['username'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/account/login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['username']))
-        d = self._postUrl(loginUrl,params)
-
-        if "Login attempt failed..." in d:
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['username']))
-            raise exceptions.FailedToLogin(url,params['username'])
-            return False
-        else:
-            return True        
-    
-    def extractChapterUrlsAndMetadata(self):
-
-        # fetch the chapter.  From that we will get almost all the
-        # metadata and chapter list
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        try:
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        h3 = soup.find('h3')
-        storya = h3.find('a',href=re.compile("^/story/\d+$"))
-        if storya : # if there's a story link in the h3 header, this is a chapter page.
-            # normalize story URL on chapter list.
-            self.story.setMetadata('storyId',storya['href'].split('/',)[2])
-            url = "http://"+self.getSiteDomain()+storya['href']
-            logger.debug("Normalizing to URL: "+url)
-            self._setURL(url)
-            try:
-                soup = bs.BeautifulSoup(self._fetchUrl(url))
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(self.url)
-                else:
-                    raise e
-
-        # if blocked, attempt login.
-        if soup.find("li",{"class":"blocked"}):
-            if self.performLogin(url): # performLogin raises
-                                       # FailedToLogin if it fails.
-                soup = bs.BeautifulSoup(self._fetchUrl(url))
-
-        # title - first h4 tag will be title.
-        titleh4 = soup.find('h4')
-        self.story.setMetadata('title', titleh4.a.string)
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/author/\d+"))
-        self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # description
-        storydiv = soup.find("div",{"id":"story"})
-        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p)
-        #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
-
-        # most of the meta data is here:
-        metap = storydiv.find("p",{"class":"meta"})
-        self.story.addToList('category',metap.find("a",href=re.compile(r"^/category/\d+")).string)
-
-        # warnings
-        # <span class="req"><a href="/help/38" title="Medium Spoilers">[!!] </a> <a href="/help/38" title="Rape/Sexual Violence">[R] </a> <a href="/help/38" title="Violence">[V] </a> <a href="/help/38" title="Child/Underage Sex">[Y] </a></span>
-        spanreq = metap.find("span",{"class":"req"})
-        if spanreq: # can be no warnings.
-            for a in spanreq.findAll("a"):
-                self.story.addToList('warnings',a['title'])
-
-        ## perhaps not the most efficient way to parse this, using
-        ## regexps for each rather than something more complex, but
-        ## IMO, it's more readable and amenable to change.
-        metastr = stripHTML(str(metap)).replace('\n',' ').replace('\t',' ')
-        #print "metap: (%s)"%metastr
-
-        m = re.match(r".*?Rating: (.+?) -.*?",metastr)
-        if m:
-            self.story.setMetadata('rating', m.group(1))
-
-        m = re.match(r".*?Genres: (.+?) -.*?",metastr)
-        if m:
-            for g in m.group(1).split(','):
-                self.story.addToList('genre',g)
-        
-        m = re.match(r".*?Characters: (.*?) -.*?",metastr)
-        if m:
-            for g in m.group(1).split(','):
-                if g:
-                    self.story.addToList('characters',g)
-        
-        m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
-        if m:
-            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))
-
-        # Updated can have more than one space after it. <shrug>
-        m = re.match(r".*?Updated: ([0-9/]+?) +-.*?",metastr) 
-        if m:
-            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y/%m/%d"))
-
-        m = re.match(r".*? - ([0-9/]+?) words.*?",metastr)
-        if m:
-            self.story.setMetadata('numWords',m.group(1))
-
-        if metastr.endswith("Complete"):
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        # get the chapter list first this time because that's how we
-        # detect the need to login.
-        storylistul = soup.find('ul',{'id':'storylist'})
-        if not storylistul:
-            # no list found, so it's a one-chapter story.
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            chapterlistlis = storylistul.findAll('li')
-            for chapterli in chapterlistlis:
-                if "blocked" in chapterli['class']:
-                    # paranoia check.  We should already be logged in by now.
-                    raise exceptions.FailedToLogin(url,self.username)
-                else:
-                    #print "chapterli.h4.a (%s)"%chapterli.h4.a
-                    self.chapterUrls.append((chapterli.h4.a.string,
-                                             u'http://%s%s'%(self.getSiteDomain(),
-                                                             chapterli.h4.a['href'])))
-        #print "self.chapterUrls:%s"%self.chapterUrls
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        return
-
-
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        span = soup.find('div', {'id' : 'storytext'})
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,span)
-
-def getClass():
-    return FicwadComSiteAdapter
-

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -1,184 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import cookielib as cl
-from datetime import datetime
-import json
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from ..bbcodeutils.bbcodeparser import bbcodeparser
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return FimFictionNetSiteAdapter
-
-class FimFictionNetSiteAdapter(BaseSiteAdapter):
-    
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','fimficnet')
-        self.story.setMetadata('storyId', self.parsedUrl.path.split('/',)[2])
-        self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
-        self.is_adult = False
-        
-    @staticmethod
-    def getSiteDomain():
-        return 'www.fimfiction.net'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway
-        return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com']
-
-    def getSiteExampleURLs(self):
-        return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
-
-    def getSiteURLPattern(self):
-        return r"http://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
-        
-    def extractChapterUrlsAndMetadata(self):
-        
-        if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
-            cookie = cl.Cookie(version=0, name='view_mature', value='true',
-                               port=None, port_specified=False,
-                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
-                               path='/story', path_specified=True,
-                               secure=False,
-                               expires=time.time()+10000,
-                               discard=False,
-                               comment=None,
-                               comment_url=None,
-                               rest={'HttpOnly': None},
-                               rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)
-        
-        try:
-            apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
-            apiData = json.loads(apiResponse)
-            
-            # Unfortunately, we still need to load the story index page to parse the characters
-            data = self._fetchUrl(self.url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-        
-        if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
-
-        if "/images/missing_story.png" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
-        
-        if "Invalid story id" in apiData.values():
-            raise exceptions.StoryDoesNotExist(self.url)
-        
-        if "This story has been marked as having adult content." in data:
-            raise exceptions.AdultCheckRequired(self.url)
-        
-        if self.password:
-            params = {}
-            params['password'] = self.password
-            data = self._postUrl(self.url,params)
-
-        if "Enter the password the author set for this story to view it." in data:
-            if self.getConfig('fail_on_password'):
-                raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
-            else:
-                raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
-         
-        storyMetadata = apiData["story"]    
-            
-        self.story.setMetadata("title", storyMetadata["title"])
-        self.story.setMetadata("author", storyMetadata["author"]["name"])
-        self.story.setMetadata("authorId", storyMetadata["author"]["id"])
-        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
-        
-        chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
-        for chapter in chapters:
-            self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
-        self.story.setMetadata("numChapters", len(self.chapterUrls))
-        
-        # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
-        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
-        # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
-        # status to leave unchanged.
-        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
-        self.story.setMetadata("status", status)
-        self.story.setMetadata("rating", storyMetadata["content_rating_text"])
-            
-        for category in storyMetadata["categories"]:
-            if storyMetadata["categories"][category]:
-                self.story.addToList("genre", category) 
-
-        self.story.setMetadata("numWords", str(storyMetadata["words"]))
-        
-        # fimfic is the first site with an explicit cover image.
-        if "image" in storyMetadata.keys():
-            if "full_image" in storyMetadata:
-                coverurl = storyMetadata["full_image"]
-            else:
-                coverurl = storyMetadata["image"]
-            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
-                coverurl = "http:"+coverurl
-            self.setCoverImage(self.url,coverurl)
-
-        # the fimfic API gives bbcode for desc, not html.
-        # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
-        self.setDescription(self.url,
-                            bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
-        
-        # Dates are in Unix time
-        # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
-        
-        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
-        # fimfic stopped putting the char name on or around the char
-        # icon now for some reason.  Pull it from the image name with
-        # some heuristics.
-        for character in [character_icon["src"] for character_icon in soup.findAll("img", {"class":"character_icon"})]:
-            # //static.fimfiction.net/images/characters/twilight_sparkle.png
-            # 5th split /, remove last four, replace _, capitolize every word(title())
-            char = character.split('/')[5][:-4].replace('_',' ').title()
-            if char == 'Oc':
-                char = "OC"
-            if char == 'Cmc':
-                char = "Cutie Mark Crusaders"
-            self.story.addToList("characters", char)
-            
-            
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
-        if soup == None:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-        return self.utf8FromSoup(url,soup)
-        

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -1,294 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return FineStoriesComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class FineStoriesComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
-        if 'storyInfo' in self.story.getMetadata('storyId'):
-            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','fnst')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y-%m-%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'finestories.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Free Registration' in data \
-                or "Invalid Password!" in data \
-                or "Invalid User Name!" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['theusername'] = self.username
-            params['thepassword'] = self.password
-        else:
-            params['theusername'] = self.getConfig("username")
-            params['thepassword'] = self.getConfig("password")
-        params['rememberMe'] = '1'
-        params['page'] = 'http://finestories.com/'
-        params['submit'] = 'Login'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['theusername']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "My Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['theusername']))
-            raise exceptions.FailedToLogin(url,params['theusername'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"/a/\w+"))
-        self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',a.text)
-
-        # Find the chapters:
-        chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$"))
-        if len(chapters) != 0:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # surprisingly, the detailed page does not give enough details, so go to author's page
-        
-        skip=0
-        i=0
-        while i == 0:
-            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
-        
-            a = asoup.findAll('td', {'class' : 'lc2'})
-            for lc2 in a:
-                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
-                    i=1
-                    break
-                if a[len(a)-1] == lc2:
-                    skip=skip+10
-        
-        for cat in lc2.findAll('div', {'class' : 'typediv'}):
-            self.story.addToList('category',cat.text)
-        
-        self.story.setMetadata('numWords', lc2.findNext('td', {'class' : 'num'}).text)
-        
-        lc4 = lc2.findNext('td', {'class' : 'lc4'})
-        
-        
-        try:
-            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
-            i = a.parent.text.split('(')[1].split(')')[0]
-            self.setSeries(a.text, i)
-        except:
-            pass
-        try:
-            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
-            self.story.addToList("category",a.text)
-        except:
-            pass
-            
-        for a in lc4.findAll('span', {'class' : 'help'}):
-            a.extract()
-            
-        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),lc4.text.split('[More Info')[0])
-            
-        for b in lc4.findAll('b'):
-            label = b.text
-            value = b.nextSibling
-            
-            if 'For Age' in label:
-                self.story.setMetadata('rating', value)
-                
-            if 'Tags' in label:
-                for genre in value.split(', '):
-                    self.story.addToList('genre',genre)
-                    
-            if 'Posted' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
-                
-            if 'Concluded' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
-                
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
-                
-        status = lc4.find('span', {'class' : 'ab'})
-        if  status != None:
-            self.story.setMetadata('status', 'In-Progress')
-            if "Last Activity" in status.text:
-                self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat))
-        else:
-            self.story.setMetadata('status', 'Completed')
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-        
-        # some big chapters are split over several pages
-        pager = div.find('span', {'class' : 'pager'})
-        if pager != None:
-            urls=pager.findAll('a')
-            urls=urls[:len(urls)-1]
-            
-            
-            for ur in urls:
-                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-                div1 = soup.find('div', {'id' : 'story'})
-                
-                # appending next section
-                last=div.findAll('p')
-                next=div1.find('span', {'class' : 'conTag'}).nextSibling
-            
-                last[len(last)-1]=last[len(last)-1].append(next)
-                div.append(div1)
-            
-            
-            
-        # removing all the left-over stuff    
-        for a in div.findAll('span'):
-            a.extract()
-            
-        for a in div.findAll('h1'):
-            a.extract()
-        for a in div.findAll('h2'):
-            a.extract()
-        for a in div.findAll('h3'):
-            a.extract()
-        for a in div.findAll('h4'):
-            a.extract()
-        for a in div.findAll('br'):
-            a.extract()
-        for a in div.findAll('div', {'class' : 'date'}):
-            a.extract()
-            
-        a = div.find('form')
-        if a != None:
-            b = a.nextSibling
-            while b != None:
-                a.extract()
-                a=b
-                b=b.nextSibling
-        
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -1,298 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return GrangerEnchantedCom
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class GrangerEnchantedCom(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.section=self.parsedUrl.path.split('/',)[1]
-             
-        # normalized story URL.
-        if "malfoymanor" in self.parsedUrl.netloc:
-            self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId'))
-            self.story.addToList("category","The Manor")
-        else:
-            self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','gech')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%b/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'grangerenchanted.com'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
-
-    def getSiteExampleURLs(self):
-        return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        if "enchant" in self.section:
-            loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
-        else:
-            loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=1"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story1'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -1,201 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','hp')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only psid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
-
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'www.harrypotterfanfiction.com'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com']
-
-    def getSiteExampleURLs(self):
-        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234 http://harrypotterfanfiction.com/viewstory.php?psid=5678"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?"+re.escape("harrypotterfanfiction.com/viewstory.php?psid=")+r"\d+$"
-
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-          return True
-        else:
-          return False
-
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
-        ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
-        if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?showuid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        ## hpcom doesn't give us total words--but it does give
-        ## us words/chapter.  I'd rather add than fetch and
-        ## parse another page.
-        words=0
-        for tr in soup.find('table',{'class':'text'}).findAll('tr'):
-            tdstr = tr.findAll('td')[2].string
-            if tdstr and tdstr.isdigit():
-                words+=int(tdstr)
-        self.story.setMetadata('numWords',str(words))
-        
-        # Find the chapters:
-        tablelist = soup.find('table',{'class':'text'})
-        for chapter in tablelist.findAll('a', href=re.compile(r'\?chapterid=\d+')):
-            #javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?chapterid=433441&i=1'
-            # just in case there's tags, like <i> in chapter titles.
-            chpt=re.sub(r'^.*?(\?chapterid=\d+).*?',r'\1',chapter['href'])
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php'+chpt))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        ## Finding the metadata is a bit of a pain.  Desc is the only thing this color.
-        desctable= soup.find('table',{'bgcolor':'#f0e8e8'})
-        self.setDescription(url,desctable)
-        #self.story.setMetadata('description',stripHTML(desctable))
-
-        ## Finding the metadata is a bit of a pain.  Most of the meta
-        ## data is in a center.table without a bgcolor.
-        for center in soup.findAll('center'):
-            table = center.find('table',{'bgcolor':None})
-            if table:
-                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-                # Rating: 12+ Story Reviews: 3
-                # Chapters: 3
-                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-                # Status: Completed
-                # First Published: 2010.09.02
-                # Last Published Chapter: 2010.09.28
-                # Last Updated: 2010.09.28
-                # Favorite Story Of: 1 users
-                # Warnings: Scenes of a Mild Sexual Nature
-
-                m = re.match(r".*?Status: Completed.*?",metastr)
-                if m:
-                    self.story.setMetadata('status','Completed')
-                else:
-                    self.story.setMetadata('status','In-Progress')
-
-                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-                if m:
-                    self.story.setMetadata('rating', m.group(1))
-
-                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('genre',g)
-        
-                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('characters',g)
-        
-                m = re.match(r".*?Warnings: (.+).*?",metastr)
-                if m:
-                    for w in m.group(1).split(','):
-                        if w != 'Now Warnings':
-                            self.story.addToList('warnings',w)
-        
-                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
-
-                # Updated can have more than one space after it. <shrug>
-                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        ## most adapters use BeautifulStoneSoup here, but non-Stone
-        ## allows nested div tags.
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'fluidtext'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)
-
-def getClass():
-    return HarryPotterFanFictionComSiteAdapter
-

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -1,229 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return HLFictionNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class HLFictionNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','hlf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'hlfiction.net'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select')
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                # just in case there's tags, like <i> in chapter titles. 
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                break
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
-        for labelspan in labels:
-            label = labelspan.string
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'classification':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
-
-            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -1,230 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return HPFandomNetAdapterAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/eff/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','hpfdm') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y.%m.%d" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.hpfandom.net' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/eff/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        ## Going to get the rest from the author page.
-        authdata = self._fetchUrl(self.story.getMetadata('authorUrl'))
-        # fix a typo in the site HTML so I can find the Characters list.
-        authdata = authdata.replace('<td width=10%">','<td width="10%">')
-
-        # hpfandom.net only seems to indicate adult-only by javascript on the story/chapter links.
-        if "javascript:if (confirm('Slash/het fiction which incorporates sexual situations to a somewhat graphic degree and some violence. ')) location = 'viewstory.php?sid=%s'"%self.story.getMetadata('storyId') in authdata \
-                and not (self.is_adult or self.getConfig("is_adult")):
-            raise exceptions.AdultCheckRequired(self.url)
-        
-        authsoup = bs.BeautifulSoup(authdata)
-
-        reviewsa = authsoup.find('a', href="reviews.php?sid="+self.story.getMetadata('storyId')+"&a=")
-        # <table><tr><td><p><b><a ...>
-        metablock = reviewsa.findParent("table")
-        #print("metablock:%s"%metablock)
-        
-        ## Title
-        titlea = metablock.find('a', href=re.compile("viewstory.php"))
-        #print("titlea:%s"%titlea)
-        if titlea == None:
-            raise exceptions.FailedToDownload("Story URL (%s) not found on author's page, can't use chapter URLs"%url)
-        self.story.setMetadata('title',stripHTML(titlea))
-        
-        # Find the chapters: !!! hpfandom.net differs from every other
-        # eFiction site--the sid on viewstory for chapters is
-        # *different* for each chapter
-        for chapter in soup.findAll('a', {'href':re.compile(r"viewstory.php\?sid=\d+&i=\d+")}):
-            m = re.match(r'.*?(viewstory.php\?sid=\d+&i=\d+).*?',chapter['href'])
-            # just in case there's tags, like <i> in chapter titles.
-            #print("====chapter===%s"%m.group(1))
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eff/'+m.group(1)))
-
-        if len(self.chapterUrls) == 0:
-            self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        summary = metablock.find("td",{"class":"summary"})
-        summary.name='span'
-        self.setDescription(url,summary)
-
-        # words & completed in first row of metablock.
-        firstrow = stripHTML(metablock.find('tr'))
-        # A Mother's Love xx Going Grey 1 (G+) by Kiristeen | Reviews - 18 | Words: 27468 | Completed: Yes
-        m = re.match(r".*?\((?P<rating>[^)]+)\).*?Words: (?P<words>\d+).*?Completed: (?P<status>Yes|No)",firstrow)
-        if m != None:
-            if m.group('rating') != None:
-                self.story.setMetadata('rating', m.group('rating'))
-                
-            if m.group('words') != None:
-                self.story.setMetadata('numWords', m.group('words'))
-
-            if m.group('status') != None:
-                if 'Yes' in m.group('status'):
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-                
-
-        # <tr><td width="10%" valign="top">Chapters:</td><td width="40%" valign="top">4</td>
-        # <td width="10%" valign="top">Published:</td><td width="40%" valign="top">2010.09.29</td></tr>
-        # <tr><td width="10%" valign="top">Completed:</td><td width="40%" valign="top">Yes</td><td width="10%" valign="top">Updated:</td><td width="40%" valign="top">2010.10.03</td></tr>
-        labels = metablock.findAll('td',{'width':'10%'})
-        for td in labels:
-            label = td.string
-            value = td.nextSibling.string
-            #print("\nlabel:%s\nvalue:%s\n"%(label,value))
-
-            if 'Category' in label:
-                cats = td.parent.findAll('a',href=re.compile(r'categories.php'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.split(','):
-                    self.story.addToList('characters',char.strip())
-
-            if 'Genre' in label:
-                for genre in value.split(','):
-                    self.story.addToList('genre',genre.strip())
-
-            if 'Warnings' in label:
-                for warning in value.split(','):
-                    if warning.strip() != 'none':
-                        self.story.addToList('warnings',warning.strip())
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        # There's no good wrapper around the chapter text. :-/
-        # There are, however, tables with width=100% just above and below the real text.
-        data = re.sub(r'<table width="100%">.*?</table>','<div name="storybody">',
-                      data,count=1,flags=re.DOTALL)
-        
-        data = re.sub(r'<table width="100%">.*?</table>','</div>',
-                      data,count=1,flags=re.DOTALL)
-        
-        soup = bs.BeautifulStoneSoup(data,selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        div = soup.find("div",{'name':'storybody'})
-        #print("\n\ndiv:%s\n\n"%div)
-        
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -1,217 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return HPFanficArchiveComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class HPFanficArchiveComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','hpffa')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.hpfanficarchive.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/stories/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/stories/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-        
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -1,281 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return IkEternalNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class IkEternalNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ike')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.ik-eternal.net'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=1"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data,selfClosingTags=('p')) #poor formatting of the paragraphs in the title page
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        asoup = soup.find('div', {'class': 'listbox'})
-        for a in asoup.findAll('p'):
-            a.name='br'
-        labels = asoup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -1,312 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return KSArchiveComAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class KSArchiveComAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ksa') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b/%d/%Y" # XXX
-            
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.ksarchive.com','ksarchive.com']
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'ksarchive.com' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-
-            # Furthermore, there's a couple sites now with more than
-            # one warning level for different ratings.  And they're
-            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
-            # we'll try 5 first.
-            addurl = "&ageconsent=ok&warning=2" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag.
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',stripHTML(a))
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    # poor HTML(unclosed <p> for one) can cause run on
-                    # over the next label.
-                    if '<span class="label">' in svalue:
-                        svalue = svalue[0:svalue.find('<span class="label">')]
-                        break
-                    else:
-                        value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [stripHTML(cat) for cat in cats]
-                for cat in catstext:
-                    # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
-                    # tag in the desc once.
-                    if cat and cat.strip() in ('Poetry','Essays'):
-                        self.story.addToList('category',stripHTML(cat))
-
-            if 'Characters' in label:
-                self.story.addToList('characters','Kirk')
-                self.story.addToList('characters','Spock')
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [stripHTML(char) for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',stripHTML(char))
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [stripHTML(genre) for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',stripHTML(genre))
-
-            ## In addition to Genre (which is very site specific) KSA
-            ## has 'Story Type', which is much more what most sites
-            ## call genre.
-            if 'Story Type' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
-                genrestext = [stripHTML(genre) for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',stripHTML(genre))
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [stripHTML(warning) for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',stripHTML(warning))
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = stripHTML(a)
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            if "A fatal MySQL error was encountered" in data:
-                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Database error on the site reported!" % url)
-            else:
-                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -1,248 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return LibraryOfMoriaComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class LibraryOfMoriaComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','lom')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.libraryofmoria.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/a/viewstory.php?sid=")+r"\d+$"
-
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            addurl = "&ageconsent=ok&warning=3"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/a/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Type' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warning' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/a/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -1,235 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return LumosSycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class LumosSycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','lsph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'lumos.sycophanthex.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=19"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Age Consent Required" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
-        
-        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        rating=pt.text.split('(')[1].split(')')[0]
-        self.story.setMetadata('rating', rating)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-				
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-		
-        labels = soup.findAll('span',{'class':'label'})
-		
-        value = labels[0].previousSibling
-        svalue = ""
-        while value != None:
-            val = value
-            value = value.previousSibling
-        while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-		
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value.split(' -')[0])
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Complete' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -1,236 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','mm')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-            # normalized story URL.
-            self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'www.mediaminer.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+self.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
-
-    def getSiteURLPattern(self):
-        ##  http://www.mediaminer.org/fanfic/view_st.php/76882
-        ##  http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
-        return re.escape("http://"+self.getSiteDomain())+\
-            "/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        # [ A - All Readers ], strip '[' ']'
-        ## Above title because we remove the smtxt font to get title.
-        smtxt = soup.find("font",{"class":"smtxt"})
-        if not smtxt:
-            raise exceptions.StoryDoesNotExist(self.url)
-        rating = smtxt.string[1:-1]
-        self.story.setMetadata('rating',rating)
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"/fanfic/src.php/u/\d+"))
-        self.story.setMetadata('authorId',a['href'].split('/')[-1])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        ## Title - Good grief.  Title varies by chaptered, 1chapter and 'type=one shot'--and even 'one-shot's can have titled chapter.
-        ## But, if colspan=2, there's no chapter title.
-        ## <td class="ffh">Atmosphere: Chapter 1</b> <font class="smtxt">[ P - Pre-Teen ]</font></td>
-        ## <td colspan=2 class="ffh">Hearts of Ice <font class="smtxt">[ P - Pre-Teen ]</font></td>
-        ## <td colspan=2 class="ffh">Suzaku no Princess <font class="smtxt">[ P - Pre-Teen ]</font></td>
-        ## <td class="ffh">The Kraut, The Bartender, and The Drunkard: Chapter 1</b> <font class="smtxt">[ P - Pre-Teen ]</font></td>
-        ## <td class="ffh">Betrayal and Justice: A Cold Heart</b> <font size="-1">( Chapter 1 )</font> <font class="smtxt">[ A - All Readers ]</font></td>
-        ## <td class="ffh">Question and Answer: Question and Answer</b> <font size="-1">( One-Shot )</font> <font class="smtxt">[ A - All Readers ]</font></td>
-        title = soup.find('td',{'class':'ffh'})
-        for font in title.findAll('font'):
-            font.extract() # removes 'font' tags from inside the td.        
-        if title.has_key('colspan'):
-            titlet = title.text
-        else:
-            ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
-        self.story.setMetadata('title',titlet)
-        ## The story title is difficult to reliably parse from the
-        ## story pages.  Getting it from the author page is, but costs
-        ## another fetch.
-        # authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        # titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')})
-        # self.story.setMetadata('title',titlea.text)
-
-        # save date from first for later.
-        firstdate=None
-        
-        # Find the chapters
-        select = soup.find('select',{'name':'cid'})
-        if not select:
-            self.chapterUrls.append(( self.story.getMetadata('title'),self.url))
-        else:
-            for option in select.findAll("option"):
-                chapter = stripHTML(option.string)
-                ## chapter can be: Chapter 7 [Jan 23, 2011]
-                ##             or: Vigilant Moonlight ( Chapter 1 ) [Jan 30, 2004]
-                ##        or even: Prologue ( Prologue ) [Jul 31, 2010]
-                m = re.match(r'^(.*?) (\( .*? \) )?\[(.*?)\]$',chapter)
-                chapter = m.group(1)
-                # save date from first for later.
-                if not firstdate:
-                    firstdate = m.group(3)
-                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # category
-        # <a href="/fanfic/src.php/a/567">Ranma 1/2</a>
-        for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/a/")):
-            self.story.addToList('category',a.string)
-        
-        # genre
-        # <a href="/fanfic/src.php/a/567">Ranma 1/2</a>
-        for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/g/")):
-            self.story.addToList('genre',a.string)
-
-        # if firstdate, then the block below will only have last updated.
-        if firstdate:
-            self.story.setMetadata('datePublished', makeDate(firstdate, "%b %d, %Y"))
-        # Everything else is in <tr bgcolor="#EEEED4">
-
-        metastr = stripHTML(soup.find("tr",{"bgcolor":"#EEEED4"})).replace('\n',' ').replace('\r',' ').replace('\t',' ')
-        # Latest Revision: August 03, 2010
-        m = re.match(r".*?(?:Latest Revision|Uploaded On): ([a-zA-Z]+ \d\d, \d\d\d\d)",metastr)
-        if m:
-            self.story.setMetadata('dateUpdated', makeDate(m.group(1), "%B %d, %Y"))
-            if not firstdate:
-                self.story.setMetadata('datePublished',
-                                       self.story.getMetadataRaw('dateUpdated'))
-                
-        else:
-            self.story.setMetadata('dateUpdated',
-                                   self.story.getMetadataRaw('datePublished'))
-
-        # Words: 123456
-        m = re.match(r".*?\| Words: (\d+) \|",metastr)
-        if m:
-            self.story.setMetadata('numWords', m.group(1))
-            
-        # Summary: ....
-        m = re.match(r".*?Summary: (.*)$",metastr) 
-        if m:
-            self.setDescription(url, m.group(1))
-            #self.story.setMetadata('description', m.group(1))
-
-        # completed
-        m = re.match(r".*?Status: Completed.*?",metastr)
-        if m:
-            self.story.setMetadata('status','Completed')
-        else:
-            self.story.setMetadata('status','In-Progress')
-
-        return
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data=self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        anchor = soup.find('a',{'name':'fic_c'})
-
-        if None == anchor:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        ## find divs with align=left, those are paragraphs in newer stories.
-        divlist = anchor.findAllNext('div',{'align':'left'})
-        if divlist:
-            for div in divlist:
-                div.name='p' # convert to <p> mediaminer uses div with
-                             # a margin for paragraphs.
-                anchor.append(div) # cheat!  stuff all the content
-                                   # divs into anchor just as a
-                                   # holder.
-                del div['style']
-                del div['align']
-            anchor.name='div'
-            return self.utf8FromSoup(url,anchor)
-        
-        else:
-            logger.debug('Using kludgey text find for older mediaminer story.')
-            ## Some older mediaminer stories are unparsable with BeautifulSoup.
-            ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
-            ## Story stuff falls between:
-            data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>"
-            soup = bs.BeautifulStoneSoup(data,
-                                         selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-            for tag in soup.findAll('td',{'class':'ffh'}) + \
-                    soup.findAll('div',{'class':'acl'}) + \
-                    soup.findAll('div',{'class':'footer smtxt'}) + \
-                    soup.findAll('table',{'class':'tbbrdr'}):
-                tag.extract() # remove tag from soup.
-                
-            return self.utf8FromSoup(url,soup)
-        
-
-def getClass():
-    return MediaMinerOrgSiteAdapter
-

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -1,291 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return MerlinFicDtwinsCoUk
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class MerlinFicDtwinsCoUk(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','mrfd')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'merlinfic.dtwins.co.uk'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-                    
-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for ship in ships:
-                    self.story.addToList('ships',ship.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -1,287 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return MidnightwhispersCaAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','mw') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.midnightwhispers.ca' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-
-            # Furthermore, there's a couple sites now with more than
-            # one warning level for different ratings.  And they're
-            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
-            # we'll try 5 first.
-            addurl = "&ageconsent=ok&warning=5" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  nfacommunity uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag.
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            if "A fatal MySQL error was encountered" in data:
-                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Database error on the site reported!" % url)
-            else:
-                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -1,329 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return MuggleNetComAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class MuggleNetComAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','mgln') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction.mugglenet.com' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if "class='errortext'>Registered Users Only" in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId')
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            # http://fanfiction.mugglenet.com/viewstory.php?sid=91079&ageconsent=ok&warning=3
-            addurl = "&ageconsent=ok&warning=3" # XXX  &warning=5
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            #print("\nurl:%s\ndata:\n%s\n"%(url,data))
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  nfacommunity uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=%s((?:&ageconsent=ok)?&warning=\d+)'"%self.story.getMetadata('storyId'),data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-
-        # Not good enough-- content can contain a ('), which ends the content prematurely.
-        # metadesc = soup.find('meta',{'name':'description'})
-        # print("removeAllEntities(metadesc['content']):\n%s\n"%removeAllEntities(metadesc['content']))
-        start='<span class="label">Summary: </span>'
-        end='<span class="label">Rated:</span>'
-        summarydata = data[data.index(start)+len(start):data.index(end)]
-        #print("summarydata:\n%s\n"%summarydata)
-        self.setDescription(url,bs.BeautifulSoup(summarydata))
-        
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            # not good enough--poorly formated summary html will break it.
-            # if 'Summary' in label:
-            #     ## Everything until the next span class='label'
-            #     svalue = ""
-            #     while not defaultGetattr(value,'class') == 'label':
-            #         svalue += str(value)
-            #         value = value.nextSibling
-            #     self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -1,209 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NationalLibraryNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NationalLibraryNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only storyid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ntlb')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m-%d-%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        return 'national-library.net'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.national-library.net','national-library.net']
-
-    def getSiteExampleURLs(self):
-        return "ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for p in soup.findAll('p'):
-            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
-            if len(chapters) > 0:
-                for chapter in chapters:
-                    # just in case there's tags, like <i> in chapter titles.
-                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-                break
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        self.story.setMetadata('status', 'Completed')
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('b')
-        for x in range(2,len(labels)):
-            value = labels[x].nextSibling
-            label = labels[x].string
-            
-            if 'Summary' in label:
-                self.setDescription(url,value)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', stripHTML(value.nextSibling))
-
-            if 'Word Count' in label:
-                self.story.setMetadata('numWords', value.string)
-
-            if 'Category' in label:
-                for cat in value.string.split(', '):
-                    self.story.addToList('category',cat)
-            if 'Crossover Shows' in label:
-                for cat in value.string.split(', '):
-                    if "No Show" not in cat:
-                        self.story.addToList('category',cat)
-
-            if 'Character' in label:
-                for char in value.string.split(', '):
-                    self.story.addToList('characters',char)
-                    
-            if 'Pairing' in label:
-                for char in value.string.split(', '):
-                    self.story.addToList('ships',char)
-                    
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    self.story.addToList('warnings',warning)
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Series' in label:
-                self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
-                
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
-            
-        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
-        for genre in a:
-            self.story.setMetadata('genre', genre)
-        
-        a=story.findNext(text=re.compile('Archived'))
-        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div')
-        
-        # bit messy since higly inconsistent
-        for p in soup.findAll('p', {'align' : 'center'}):
-            p.extract()
-        p = soup.findAll('p')
-        for x in range(0,3):
-            p[x].extract()
-        if "Chapters: " in stripHTML(p[3]):
-            p[3].extract()
-        for x in range(len(p)-2,len(p)-1):
-            p[x].extract()
-
-        for p in soup.findAll('h1'):
-            p.extract()
-        for p in soup.findAll('h3'):
-            p.extract()
-        for p in soup.findAll('a'):
-            p.extract()
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -1,216 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NCISFicComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NCISFicComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only storyid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ncisf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m-%d-%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        return 'ncisfic.com'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.ncisfic.com','ncisfic.com']
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for p in soup.findAll('p'):
-            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
-            if len(chapters) > 0:
-                for chapter in chapters:
-                    # just in case there's tags, like <i> in chapter titles.
-                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-                break
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        self.story.setMetadata('status', 'Completed')
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('b')
-        for x in range(2,len(labels)):
-            value = labels[x].nextSibling
-            label = labels[x].string
-            
-            if 'Summary' in label:
-                self.setDescription(url,value)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', stripHTML(value.nextSibling))
-
-            if 'Word Count' in label:
-                self.story.setMetadata('numWords', value.string)
-
-            if 'Category' in label:
-                for cat in value.string.split(', '):
-                    self.story.addToList('category',cat)
-            if 'Crossover Shows' in label:
-                for cat in value.string.split(', '):
-                    if "No Show" not in cat:
-                        self.story.addToList('category',cat)
-
-            if 'Character' in label:
-                for char in value.string.split(', '):
-                    self.story.addToList('characters',char)
-                    
-            if 'Pairing' in label:
-                for char in value.string.split(', '):
-                    self.story.addToList('ships',char)
-                    
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    self.story.addToList('warnings',warning)
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Series' in label:
-                if "No Series" not in value.nextSibling.string:
-                    self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
-                
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
-        
-        a=story.findNext('font')
-        if 'Complete' in a.string:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-            
-        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
-        for genre in a:
-            self.story.setMetadata('genre', genre)
-        
-        a=story.findNext(text=re.compile('Archived'))
-        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div')
-        
-        # bit messy since higly inconsistent
-        for p in soup.findAll('p', {'align' : 'center'}):
-            p.extract()
-        p = soup.findAll('p')
-        for x in range(0,3):
-            p[x].extract()
-        if "Chapters: " in stripHTML(p[3]):
-            p[3].extract()
-        for x in range(len(p)-2,len(p)-1):
-            p[x].extract()
-
-        for p in soup.findAll('h1'):
-            p.extract()
-        for p in soup.findAll('h3'):
-            p.extract()
-        for p in soup.findAll('a'):
-            p.extract()
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ncisfictioncom.py

@@ -1,201 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NCISFictionComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NCISFictionComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["iso-8859-1",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
-                         +"/chapters.php?stid="+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ncisfn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.ncisfiction.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
-
-    def getSiteURLPattern(self):
-        return "http://"+self.getSiteDomain()+r'/(chapters|story)?.php\?stid=\d+'
-       
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulStoneSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'class' : 'main_title'})
-        
-        aut = a.find('a')
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        
-        aut.extract()
-        self.story.setMetadata('title',stripHTML(a)[:len(stripHTML(a))-2])
-
-        # Find the chapters:
-        i=0
-        chapters=soup.findAll('table', {'class' : 'story_table'})
-        for chapter in chapters:
-            ch=chapter.find('a')
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/'+ch['href']))
-            if i == 0:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
-            if i == len(chapters)-1:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
-            i=i+1
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        info = soup.find('table', {'class' : 'story_info'})
-        
-        # no convenient way to calculate word count as it is logged differently for stories with and without series
-        
-        labels = info.findAll('tr')
-        for tr in labels:
-            value = tr.find('td')
-            label = tr.find('th').string
-
-            if 'Summary' in label:
-                self.setDescription(url,value)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.string)
-
-            if 'Category' in label:
-                cats = value.findAll('a')
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = value.findAll('a')
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Pairing' in label:
-                ships = value.findAll('a')
-                for ship in ships:
-                    self.story.addToList('ships',ship.string)
-
-            if 'Genre' in label:
-                genres = value.findAll('a')
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = value.findAll('a')
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Status' in label:
-                if 'not completed' in value.text:
-                    self.story.setMetadata('status', 'In-Progress')
-                else:
-                    self.story.setMetadata('status', 'Completed')
-                    
-        try:
-            # Find Series name from series URL.
-            a = soup.find('div',{'class' : 'sub_header'})
-            series_name = a.find('a').string
-            i = a.text.split('#')[1]
-            self.setSeries(series_name, i)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'class' : 'story_text'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -1,287 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return NfaCommunityComAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nfa') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y" # XXX
-            
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.nfacommunity.com','nfacommunity.com']
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'nfacommunity.com' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-
-            # Furthermore, there's a couple sites now with more than
-            # one warning level for different ratings.  And they're
-            # fussy about it.  nfacommunity has two: 4 & 5.
-            # we'll try 5 first.
-            addurl = "&ageconsent=ok&warning=5" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  nfacommunity uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -1,212 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NHAMagicalWorldsUsAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nha')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'nha.magical-worlds.us'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                self.story.setMetadata('title',a.string)
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'br':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-        
-        #does not provide convenient way to get word count
-        labels = info.findAll('i')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.split(' -')[0])
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if 'None' not in genre:
-                        self.story.addToList('genre',genre.split(' -')[0])
-            
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if 'None' not in char:
-                        self.story.addToList('characters',char.split(' -')[0])
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if 'None' not in warning:
-                        self.story.addToList('warnings',warning.split(' -')[0])
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"id" : "story"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -1,261 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return OcclumencySycophantHexComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','osph')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'occlumency.sycophanthex.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'This story contains adult content and/or themes.' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Logout" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'class' : 'border'}):
-            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                self.story.setMetadata('title',a.text)
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-			
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'table':
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = info.findAll('b')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word Count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if genre != 'none':
-                        self.story.addToList('genre',genre)
-						
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if char != 'none':
-                        self.story.addToList('characters',char)
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if warning != ' none':
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-        data = data.replace('<div align="left"', '<div align="left">')
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr'))
-        
-        story = soup.find('div', {"align" : "left"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -1,267 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return OneDirectionFanfictionComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','odf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'onedirectionfanfiction.com'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']
-
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-        
-        ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-    
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-        
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Age Consent Required" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -1,238 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2, urllib, cookielib
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PhoenixSongNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PhoenixSongNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','phs')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.phoenixsong.net'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/story/")+r"\d+/?$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Please login to continue.' in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-		
-        if self.password:
-            params['txtusername'] = self.username
-            params['txtpassword'] = self.password
-        else:
-            params['txtusername'] = self.getConfig("username")
-            params['txtpassword'] = self.getConfig("password")
-        #params['remember'] = '1'
-        params['login'] = 'Login'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/users/processlogin.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['txtusername']))
-        d = self._fetchUrl(loginUrl, params)
-    
-        if 'Please login to continue.' in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['txtusername']))
-            raise exceptions.FailedToLogin(url,params['txtusername'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-			
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        b = soup.find('div', {'id' : 'nav25'})
-        a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
-        a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        chapters = soup.find('select')
-        if chapters == None:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-            for b in soup.findAll('b'):
-                if b.text == "Updated":
-                    date = b.nextSibling.string.split(': ')[1].split(',')
-                    self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
-                    self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
-        else:
-            i = 0
-            chapters = chapters.findAll('option')
-            for chapter in chapters:
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['value']))
-                if i == 0:
-                    self.story.setMetadata('storyId',chapter['value'].split('/')[3])
-                    head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b')
-                    for b in head:
-                        if b.text == "Updated":
-                            date = b.nextSibling.string.split(': ')[1].split(',')
-                            self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
-                            
-                if  i == (len(chapters)-1):
-                    head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b')
-                    for b in head:
-                        if b.text == "Updated":
-                            date = b.nextSibling.string.split(': ')[1].split(',')
-                            self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
-                i = i+1
-				
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))        	
-				
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        info = asoup.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        while info != None:
-            info = info.findNext('div')
-            b = info.find('b')
-            val = b.nextSibling
-            
-            if 'Rating' in b.string:
-                self.story.setMetadata('rating', val.string.split(': ')[1])
-				
-            if 'Words' in b.string:
-                self.story.setMetadata('numWords', val.string.split(': ')[1])
-				
-            if 'Setting' in b.string:
-                self.story.addToList('category', val.string.split(': ')[1])
-				
-            if 'Status' in b.string:
-                if 'Completed' in val:
-                    val = 'Completed'
-                else:
-                    val = 'In-Progress'
-                self.story.setMetadata('status', val)
-            
-            if 'Summary' in b.string:
-                b.extract()
-                info.find('br').extract()
-                self.setDescription(url,info)
-                break
-	
-	
-	# grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
-        for p in soup.findAll('p'):
-            if "This is for problems with the formatting or the layout of the chapter." in stripHTML(p):
-                break
-            chapter.append(p)
-			
-        for a in chapter.findAll('div'):
-            a.extract()
-        for a in chapter.findAll('table'):
-            a.extract()
-        for a in chapter.findAll('script'):
-            a.extract()
-        for a in chapter.findAll('form'):
-            a.extract()
-        for a in chapter.findAll('textarea'):
-            a.extract()
-
-
-        if None == chapter:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,chapter)

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -1,248 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PonyFictionArchiveNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        logger.info(self.parsedUrl.netloc)
-        # normalized story URL.
-        if "explicit" in self.parsedUrl.netloc:
-            self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-            self.dateformat = "%d/%b/%y"
-        else:
-            self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-            self.dateformat = "%d %b %Y"
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pffa')
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'ponyfictionarchive.net'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=9"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-           
-        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-        for warning in warnings:
-            self.story.addToList('warnings',warning.string)
-        
-        status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-        self.story.setMetadata('status',status.string)
-        
-        section = soup.findAll('span', {'class' : 'General'})[1]            
-        
-        self.story.setMetadata('rating', section.previousSibling.previousSibling.string)
-
-        value = section.nextSibling
-        svalue = ""
-        while not defaultGetattr(value,'class') == 'label':
-            svalue += str(value)
-            value = value.nextSibling
-        self.setDescription(url,svalue)
-        
-        # <span class="label">Rated:</span> NC-17<br /> etc           
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -1,274 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import cookielib as cl
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return PortkeyOrgAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','prtky') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction.portkey.org' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story/1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story/")+r"\d+(/\d+)?$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        # portkey screws around with using a different URL to set the
-        # cookie and it's a pain.  So... cheat!
-        if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
-            cookie = cl.Cookie(version=0, name='verify17', value='1',
-                               port=None, port_specified=False,
-                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
-                               path='/', path_specified=True,
-                               secure=False,
-                               expires=time.time()+10000,
-                               discard=False,
-                               comment=None,
-                               comment_url=None,
-                               rest={'HttpOnly': None},
-                               rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)        
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if "You must be over 18 years of age to view it" in data: # XXX 
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        #print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"/profile/\d+"))
-        #print("======a:%s"%a)
-        self.story.setMetadata('authorId',a['href'].split('/')[-1])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        ## Going to get the rest from the author page.
-        authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        ## Title
-        titlea = authsoup.find('a', href=re.compile(r'/story/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(titlea))
-        metablock = titlea.parent
-        
-        # Find the chapters:
-        for chapter in soup.find('select',{'name':'select5'}).findAll('option', {'value':re.compile(r'/story/'+self.story.getMetadata('storyId')+"/\d+$")}):
-            # just in case there's tags, like <i> in chapter titles.
-            chtitle = stripHTML(chapter)
-            if not chtitle:
-                chtitle = "(Untitled Chapter)"
-            self.chapterUrls.append((chtitle,'http://'+self.host+chapter['value']))
-            
-        if len(self.chapterUrls) == 0:
-            self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-        # <SPAN class="dark-small-bold">Contents:</SPAN> <SPAN class="small-grey">NC17 </SPAN> 
-        # <SPAN class="dark-small-bold">Published: </SPAN><SPAN class="small-grey">12/11/07</SPAN>
-        # <SPAN class="dark-small-bold"><BR>
-        # Description:</SPAN> <SPAN class="small-black">A special book helps Harry tap into the power the Dark Lord knows not.  Of course it’s a book on sex magic and rituals… but Harry’s not complaining.  Spurned on by the ghost of a pervert founder, Harry leads his friends in the hunt for Voldemort’s Horcruxes.
-        # EROTIC COMEDY!  Loads of crude humor and sexual situations!
-        # </SPAN>
-        labels = metablock.findAll('span',{'class':'dark-small-bold'})
-        for labelspan in labels:
-            value = labelspan.findNext('span').string
-            label = stripHTML(labelspan)
-#            print("\nlabel:%s\nlabel:%s\nvalue:%s\n"%(labelspan,label,value))
-
-            if 'Description' in label:
-                self.setDescription(url,value)
-
-            if 'Contents' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Words' in label:
-                self.story.setMetadata('numWords', value)
-
-            # if 'Categories' in label:
-            #     cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-            #     catstext = [cat.string for cat in cats]
-            #     for cat in catstext:
-            #         self.story.addToList('category',cat.string)
-
-            # if 'Characters' in label:
-            #     chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-            #     charstext = [char.string for char in chars]
-            #     for char in charstext:
-            #         self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                # genre is typo'ed on the site--it falls between the
-                # dark-small-bold label and dark-small-bold content
-                # spans.
-                svalue = ""
-                value = labelspan.nextSibling
-                while not defaultGetattr(value,'class') == 'dark-small-bold':
-                    svalue += str(value)
-                    value = value.nextSibling
-                    
-                for genre in svalue.split("/"):
-                    genre = genre.strip()
-                    if genre != 'None':
-                        self.story.addToList('genre',genre)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            # if 'Warnings' in label:
-            #     warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-            #     warningstext = [warning.string for warning in warnings]
-            #     self.warning = ', '.join(warningstext)
-            #     for warning in warningstext:
-            #         self.story.addToList('warnings',warning.string)
-
-            if 'Status' in label:
-                if 'Completed' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        # try:
-        #     # Find Series name from series URL.
-        #     a = metablock.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-        #     series_name = a.string
-        #     series_url = 'http://'+self.host+'/'+a['href']
-
-        #     # use BeautifulSoup HTML parser to make everything easier to find.
-        #     seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-        #     storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-        #     i=1
-        #     for a in storyas:
-        #         if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-        #             self.setSeries(series_name, i)
-        #             break
-        #         i+=1            
-        # except:
-        #     # I find it hard to care if the series parsing fails
-        #     pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-        data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        #print("soup:%s"%soup)
-        tag = soup.find('td', {'class' : 'story'})
-        if tag == None and "<center><b>Chapter does not exist!</b></center>" in data:
-            print("Chapter is missing at: %s"%url)
-            return  self.utf8FromSoup(url,bs.BeautifulStoneSoup("<div><p><center><b>Chapter does not exist!</b></center></p><p>Chapter is missing at: <a href='%s'>%s</a></p></div>"%(url,url)))
-        tag.name='div' # force to be a div to avoid problems with nook.
-
-        centers = tag.findAll('center')
-        # first two and last two center tags are some script, 'report
-        # story', 'report story' and an ad.
-        centers[0].extract()
-        centers[1].extract()
-        centers[-1].extract()
-        centers[-2].extract()
-
-        if None == tag:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,tag)

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -1,209 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','pns')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'www.potionsandsnitches.net'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.potionsandsnitches.net','potionsandsnitches.net']
-
-    def getSiteExampleURLs(self):
-        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234 http://potionsandsnitches.net/fanfiction/viewstory.php?sid=5678"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        ## <meta name='description' content='&lt;p&gt;Description&lt;/p&gt; ...' >
-        ## Summary, strangely, is in the content attr of a <meta name='description'> tag
-        ## which is escaped HTML.  Unfortunately, we can't use it because they don't
-        ## escape (') chars in the desc, breakin the tag.
-        #meta_desc = soup.find('meta',{'name':'description'})
-        #metasoup = bs.BeautifulStoneSoup(meta_desc['content'])
-        #self.story.setMetadata('description',stripHTML(metasoup))
-
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-        
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next div class='listbox'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'listbox':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    if "Snape and Harry (required)" in char:
-                        self.story.addToList('characters',"Snape")
-                        self.story.addToList('characters',"Harry")
-                    else:
-                        self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%b %d %Y"))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-        
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)
-
-def getClass():
-    return PotionsAndSnitchesNetSiteAdapter
-

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -1,251 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PretenderCenterComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PretenderCenterComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ptdc')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'pretendercentre.com'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.pretendercentre.com','pretendercentre.com']
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/missingpieces/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/missingpieces/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/missingpieces/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.findAll('div', {'id' : 'story'})[1]
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_prisonbreakficnet.py

@@ -1,216 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return PrisonBreakFicNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PrisonBreakFicNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pbf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.prisonbreakfic.net'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
-                for ship in ships:
-                    self.story.addToList('ships',ship.string.replace(" and ","/"))
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_qafficcom.py

@@ -1,262 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return QafFicComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class QafFicComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','atp')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.qaf-fic.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=")+r"\d+$"
-       
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=NC-17"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\s+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/atp/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select')
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                # just in case there's tags, like <i> in chapter titles. 
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                break
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
-        for labelspan in labels:
-            label = labelspan.string
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'classification' and value != None:
-                    if "Featured Stories" not in value:
-                        svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
-
-            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            if list.find('a', href=re.compile(r"series.php")) != None:
-                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
-                    # Find Series name from series URL.
-                    series_url = 'http://'+self.host+'/atp/'+series['href']
-                    # use BeautifulSoup HTML parser to make everything easier to find.
-                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-                    i=1
-                    for a in storyas:
-                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                            name=seriessoup.find('div', {'id' : 'pagetitle'})
-                            name.find('a').extract()
-                            self.setSeries(name.text.split(' by[')[0], i)
-                            i=0
-                            break
-                        i+=1
-                    if i == 0:
-                        break
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-                                     
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -1,230 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return SamDeanArchiveNuAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class SamDeanArchiveNuAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','sda')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'samdean.archive.nu'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.samdean.archive.nu','samdean.archive.nu']
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select')
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                # just in case there's tags, like <i> in chapter titles. http://samdean.archive.nu/viewstory.php?sid=4317&chapter=2
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                break
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
-        for labelspan in labels:
-            label = labelspan.string
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'classification':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
-
-            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -1,245 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ScarvesAndCoffeeNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','scacf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.scarvesandcoffee.net'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=20"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('genre',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -1,256 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return SG1HeliopolisComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class SG1HeliopolisComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        self.section=self.parsedUrl.path.split('/',)[1]
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','sghp')
-
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.  Can't use extracategories, could be Atlantis or SG-1
-        if 'atlantis' in self.section:
-            self.story.addToList("category","Stargate: Atlantis")
-        else:
-            self.story.addToList("category","Stargate: SG-1")
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y.%m.%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'sg1-heliopolis.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return r"http://sg1-heliopolis.com/(archive|adult|atlantis)?/viewstory.php\?sid=\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -1,249 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return SinfulDesireOrgAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class SinfulDesireOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','snds')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.sinful-desire.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/archive/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -1,243 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return SiyeCoUkAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8",]# 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','siye') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y.%m.%d" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.siye.co.uk' # XXX
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.siye.co.uk','siye.co.uk']
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?siye\.co\.uk/(siye/)?"+re.escape("viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        # Except it doesn't this time. :-/
-        url = self.url #+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/siye/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # need(or easier) to pull other metadata from the author's list page.
-        authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-
-        ## Title
-        titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',titlea.string)
-        
-        # Find the chapters (from soup, not authsoup):
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href']))
-
-        if self.chapterUrls:
-            self.story.setMetadata('numChapters',len(self.chapterUrls))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-            self.story.setMetadata('numChapters',1)
-
-        # The stuff we can get from the chapter list/one-shot page are
-        # in the first table with 95% width.
-        metatable = soup.find('table',{'width':'95%'})
-
-        # Categories
-        cat_as = metatable.findAll('a', href=re.compile(r'categories.php'))
-        for cat_a in cat_as:
-            self.story.addToList('category',stripHTML(cat_a))
-
-        moremetaparts = stripHTML(metatable).split('\n')
-        for part in moremetaparts:
-            part = part.strip()
-            if part.startswith("Characters:"):
-                part = part[part.find(':')+1:]
-                for item in part.split(','):
-                    if item.strip() == "Harry/Ginny":
-                        self.story.addToList('characters',"Harry")
-                        self.story.addToList('characters',"Ginny")
-                    elif item.strip() not in ("None","All"):
-                        self.story.addToList('characters',item)
-
-            if part.startswith("Genres:"):
-                part = part[part.find(':')+1:]
-                for item in part.split(','):
-                    if item.strip() != "None":
-                        self.story.addToList('genre',item)
-
-            if part.startswith("Warnings:"):
-                part = part[part.find(':')+1:]
-                for item in part.split(','):
-                    if item.strip() != "None":
-                        self.story.addToList('warnings',item)
-
-            if part.startswith("Rating:"):
-                part = part[part.find(':')+1:]
-                self.story.setMetadata('rating',part)
-
-            if part.startswith("Summary:"):
-                part = part[part.find(':')+1:]
-                self.setDescription(url,part)
-                #self.story.setMetadata('description',part)
-                
-        # want to get the next tr of the table.
-        #print("%s"%titlea.parent.parent.findNextSibling('tr'))
-            
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-                
-        # SIYE formats stories in the author list differently when their part of a series.
-        # Look for non-series...
-        divdesc = titlea.parent.parent.find('div',{'class':'desc'})
-        if not divdesc:
-            # ... now look for series.
-            divdesc = titlea.parent.parent.findNextSibling('tr').find('div',{'class':'desc'})
-
-        moremeta = stripHTML(divdesc)
-        #print("moremeta:%s"%moremeta)
-        for part in moremeta.replace(' - ','\n').split('\n'):
-            #print("part:%s"%part)
-            try:
-                (name,value) = part.split(': ')
-            except:
-                # not going to worry about fancier processing for the bits
-                # that don't match.
-                continue
-            name=name.strip()
-            value=value.strip()
-            if name == 'Published':
-                self.story.setMetadata('datePublished', makeDate(value, self.dateformat))
-            if name == 'Updated':
-                self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
-            if name == 'Completed':
-                if value == 'Yes':
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-            if name == 'Words':
-                self.story.setMetadata('numWords', value)
-
-        try:
-            # Find Series name from series URL.
-            a = titlea.findPrevious('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass        
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        # soup = bs.BeautifulSoup(self._fetchUrl(url))
-        # BeautifulSoup objects to <p> inside <span>, which
-        # technically isn't allowed.
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        # not the most unique thing in the world, but it appears to be
-        # the best we can do here.
-        story = soup.find('span', {'style' : 'font-size: 100%;'})
-        
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -1,237 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-def getClass():
-    return SquidgeOrgPejaAdapter
-
-## XXX IMPORTANT NOTE!!  This adapter is for squidge.org/peja ONLY!
-## There are lots of other sites and stuff under squidge.org that
-## we're not supporting.  If/When we ever want to support more
-## sections of squidge.org, FFDL will need to be changed more
-## fundamentally to find different adapters under the same domain.
-##
-## For now, I've only implemented the part for ini section names so
-## if/when more adapters under squidge.org come along, existing ini
-## files will still work correctly.
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class SquidgeOrgPejaAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','wwomb')
-        
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.squidge.org'
-    
-    @classmethod # must be @staticmethod, don't remove it.
-    def getConfigSection(cls):
-        # The config section name.  Only override if != site domain.
-        return cls.getSiteDomain()+'/peja'
-    
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/")+"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        data = self._fetchUrl(url)
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        # Find authorid and URL from... author url.
-        author = soup.find('div', {'id':"pagetitle"}).find('a')
-        self.story.setMetadata('authorId',author['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/peja/cgi-bin/'+author['href'])
-        self.story.setMetadata('author',author.string)
-		
-        authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')})
-        self.story.setMetadata('title',title.string)
-        titleblock=title.parent.parent
-        
-        chapterselect=soup.find('select',{'name':'chapter'})
-        if chapterselect:
-            for ch in chapterselect.findAll('option'):
-                self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
-        else:
-            self.chapterUrls.append((title,url))
-		
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="classification">Rated:</span> NC-17<br /> etc
-        labels = titleblock.findAll('span',{'class':'classification'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'classification':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                if value.endswith("["):
-                    value = value[:-1]
-                self.story.setMetadata('rating', value)
-
-            if 'Characters' in label:
-                for char in value.split(','):
-                    self.story.addToList('characters',char.strip())
-
-            if 'Genre' in label:
-                for genre in value.split(','):
-                    if genre.strip() != "None":
-                        self.story.addToList('genre',genre.strip())
-
-            if 'Warnings' in label:
-                for warning in value.split(','):
-                    if warning.strip() != 'None':
-                        self.story.addToList('warnings',warning.strip())
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Fandoms' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            # http://www.squidge.org/peja/cgi-bin/series.php?seriesid=254
-            a = titleblock.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/peja/cgi-bin/'+a['href']
-    
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-                
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-	
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        chaptext = soup.find('div',{'id':"story"}).find('span')
-
-        if None == chaptext:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,chaptext)

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -1,227 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return StargateAtlantisOrgAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class StargateAtlantisOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','stat')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'stargate-atlantis.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfics/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.findAll('div', {'class' : 'content'})
-        if len(chapters) > 1:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles. 
-                link = chapter.find('a')
-                self.chapterUrls.append((stripHTML(link),'http://'+self.host+'/fanfics/'+link['href']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                break
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
-        for labelspan in labels:
-            label = labelspan.string
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'tail' and value != None:
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
-
-            if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
-
-            if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/fanfics/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-                                     
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -1,148 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return StoriesOfArdaComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class StoriesOfArdaComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','soa')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.storiesofarda.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        a = soup.find('th', {'colspan' : '3'})
-        
-        aut = a.find('a')
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        a.find('em').extract()
-        self.story.setMetadata('title',a.text)
-
-        # Find the chapters: chapterview.asp?sid=7000&cid=30919
-        chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))
-        if len(chapters)==1:
-            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
-        else:
-            for chapter in chapters:
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        summary = soup.find('td', {'colspan' : '3'})
-        self.setDescription(url,summary)
-        
-        # no convenient way to get word count
-
-        for td in asoup.findAll('td', {'colspan' : '3'}):
-            if td.find('a', href=re.compile('chapterlistview.asp\?SID='+self.story.getMetadata('storyId'))) != None:
-                break     
-        td=td.nextSibling.nextSibling
-        self.story.setMetadata('dateUpdated', makeDate(stripHTML(td).split(': ')[1], self.dateformat))
-        tr=td.parent.nextSibling.nextSibling.nextSibling.nextSibling
-        td=tr.findAll('td')
-        self.story.setMetadata('rating', td[0].string.split(': ')[1])
-        self.story.setMetadata('status', td[2].string.split(': ')[1])
-        self.story.setMetadata('datePublished', makeDate(stripHTML(td[4]).split(': ')[1], self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('table', {'width' : '90%'}).find('td')
-        div.name='div'
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_svufictioncom.py

@@ -1,272 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return SVUFictionComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class SVUFictionComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','svuf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'svufiction.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-    
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=6"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-                
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-                
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pt = soup.find('div', {'class' : 'title'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        if a.text != "":
-            self.story.setMetadata('title',a.text)
-        else:
-            self.story.setMetadata('title','Banner')
-        
-        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        rating=pt.text.split('[')[1].split(']')[0]
-        self.story.setMetadata('rating', rating)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        bottom=soup.find('div', {'class' : 'bottom'}).text
-        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
-        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
-
-        
-        content=soup.find('div', {'class' : 'content'})
-        value=content.find('h1').nextSibling
-        svalue = ""
-        while not defaultGetattr(value,'class') == 'smaller':
-            svalue += str(value)
-            value = value.nextSibling
-        self.setDescription(url,svalue)
-        
-        status = content.text.split('Completed: ')[1]
-        if 'Yes' in status:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-        
-        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-            
-        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-            
-        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-            
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -1,247 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','thpc')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        self.dateformat = "%b %d %Y"
-
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'fanfiction.tenhawkpresents.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-          return True
-        else:
-          return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            addurl = "&ageconsent=ok&warning=3"
-        else:
-            addurl=""
-            
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            addurl = "&ageconsent=ok&warning=4"
-            url = self.url+'&index=1'+addurl
-            logger.debug("Changing URL: "+url)
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        if "This story contains mature content which may include violence, sexual situations, and coarse language" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-        
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
-
-def getClass():
-    return TenhawkPresentsComSiteAdapter
-

fanficdownloader/adapters/adapter_test1.py

@@ -1,236 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import datetime
-import time
-import logging
-logger = logging.getLogger(__name__)
-
-from .. import BeautifulSoup as bs
-from .. import exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class TestSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','tst1')
-        self.crazystring = u" crazy tests:[bare amp(&) quote(') amp(&) gt(>) lt(<) ATnT(AT&T) pound(£)]"
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        self.username=''
-        self.is_adult=False
-
-    @staticmethod
-    def getSiteDomain():
-        return 'test1.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"?sid=1234"
-
-    def getSiteURLPattern(self):
-        return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$'
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.story.getMetadata('storyId') == '665' and not (self.is_adult or self.getConfig("is_adult")):
-            logger.warn("self.is_adult:%s"%self.is_adult)
-            raise exceptions.AdultCheckRequired(self.url)
-
-        if self.story.getMetadata('storyId') == '666':
-            raise exceptions.StoryDoesNotExist(self.url)
-
-        if self.story.getMetadata('storyId').startswith('670'):
-            time.sleep(1.0)
-            
-        if self.story.getMetadata('storyId').startswith('671'):
-            time.sleep(1.0)
-            
-        if self.getConfig("username"):
-            self.username = self.getConfig("username")
-        
-        if self.story.getMetadata('storyId') == '668' and self.username != "Me" :
-            raise exceptions.FailedToLogin(self.url,self.username)
-
-        if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
-            self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)')
-        else:
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
-            self.story.setMetadata('author','Test Author aa')
-        self.story.setMetadata('storyUrl',self.url)
-        self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
-<p>
-Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
-''')
-        self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
-        if self.story.getMetadata('storyId') == '669':
-            self.story.setMetadata('dateUpdated',datetime.datetime.now())
-        else:
-            self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
-        self.story.setMetadata('numWords','123456')
-
-        idnum = int(self.story.getMetadata('storyId'))
-        if idnum % 2 == 1:
-            self.story.setMetadata('status','In-Progress')
-        else:
-            self.story.setMetadata('status','Completed')
-
-        # greater than 10, no language or series.
-        if idnum < 10:
-            langs = {
-                0:"English",
-                1:"Russian",
-                2:"French",
-                3:"German",
-                }
-            self.story.setMetadata('language',langs[idnum%len(langs)])
-            self.setSeries('The Great Test',idnum)
-        
-        self.story.setMetadata('rating','Tweenie')
-        
-
-        if self.story.getMetadata('storyId') == '673':
-            self.story.addToList('author','Author From List')
-            self.story.addToList('author','Author From List 2')
-        
-            self.story.addToList('authorId','98765')
-            self.story.addToList('authorId','98765-2')
-        
-            self.story.addToList('authorUrl','http://author/url')
-            self.story.addToList('authorUrl','http://author/url-2')
-        else:
-            self.story.setMetadata('authorId','98765')
-            self.story.setMetadata('authorUrl','http://author/url')
-
-        self.story.addToList('warnings','Swearing')
-        self.story.addToList('warnings','Violence')
-
-        self.story.addToList('category','Harry Potter')
-        self.story.addToList('category','Furbie')
-        self.story.addToList('category','Crossover')
-        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
-        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
-
-        self.story.addToList('genre','Fantasy')
-        self.story.addToList('genre','Comedy')
-        self.story.addToList('genre','SF')
-        self.story.addToList('genre','Noir')
-        
-        self.story.addToList('characters','Bob Smith')
-        self.story.addToList('characters','George Johnson')
-        self.story.addToList('characters','Fred Smythe')
-        
-        self.story.addToList('listX','xVal1')
-        self.story.addToList('listX','xVal2')
-        self.story.addToList('listX','xVal3')
-        self.story.addToList('listX','xVal4')
-        
-        self.story.addToList('listY','yVal1')
-        self.story.addToList('listY','yVal2')
-        self.story.addToList('listY','yVal3')
-        self.story.addToList('listY','yVal4')
-        
-        self.story.addToList('listZ','zVal1')
-        self.story.addToList('listZ','zVal2')
-        self.story.addToList('listZ','zVal3')
-        self.story.addToList('listZ','zVal4')
-        
-        self.story.setMetadata('metaA','98765')
-        self.story.setMetadata('metaB','01245')
-        self.story.setMetadata('metaC','The mighty metaC!')
-
-        self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
-                            ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
-                            ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"),
-                            ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
-                            ('Chapter 4',self.url+"&chapter=5"),
-                            ('Chapter 5',self.url+"&chapter=6"),
-                            #('Chapter 6',self.url+"&chapter=7"),
-                            #('Chapter 7',self.url+"&chapter=8"),
-                            #('Chapter 8',self.url+"&chapter=9"),
-                            #('Chapter 9',self.url+"&chapter=0"),
-                            #('Chapter 0',self.url+"&chapter=a"),
-                            #('Chapter a',self.url+"&chapter=b"),
-                            #('Chapter b',self.url+"&chapter=c"),
-                            #('Chapter c',self.url+"&chapter=d"),
-                            #('Chapter d',self.url+"&chapter=e"),
-                            #('Chapter e',self.url+"&chapter=f"),
-                            #('Chapter f',self.url+"&chapter=g"),
-                            #('Chapter g',self.url+"&chapter=h"),
-                            #('Chapter h',self.url+"&chapter=i"),
-                            #('Chapter i',self.url+"&chapter=j"),
-                            #('Chapter j',self.url+"&chapter=k"),
-                            #('Chapter k',self.url+"&chapter=l"),
-                            #('Chapter l',self.url+"&chapter=m"),
-                            #('Chapter m',self.url+"&chapter=n"),
-                            #('Chapter n',self.url+"&chapter=o"),
-                            ]
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-                            
-
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        if self.story.getMetadata('storyId') == '667':
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
-
-        if self.story.getMetadata('storyId').startswith('670') or \
-                self.story.getMetadata('storyId').startswith('672'):
-            time.sleep(1.0)
-
-        if "chapter=1" in url :
-            text=u'''
-<div>
-<h3>Prologue</h3>
-<p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
-<p>http://test1.com?sid=664 - Crazy string title</p>
-<p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
-<p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
-<p>http://test1.com?sid=667 - raises FailedToDownload on chapter 1</p>
-<p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
-<p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
-<p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
-<p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
-<p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
-<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
-<p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
-</div>
-'''
-        else:
-            text=u'''
-<div>
-<h3>Chapter title from site</h3>
-<p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
-br breaks<br><br>
-Puella Magi Madoka Magica/魔法少女まどか★マギカ
-<!-- a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim &amp; Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a --><br/>
-br breaks<br><br>
-Don't&#8212e;ver&#8212d;o&#8212;that&#8212a;gain, &#27861; &#xE9;
-<hr>
-horizontal rules
-<hr size=1 noshade>
-<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
-<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
-</div>
-'''
-        soup = bs.BeautifulStoneSoup(text,selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        return self.utf8FromSoup(url,soup)
-
-def getClass():
-    return TestSiteAdapter
-

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -1,212 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return TheAlphaGateComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class TheAlphaGateComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','tag')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.thealphagate.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -1,204 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return TheHexFilesNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class TheHexFilesNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','thf')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y.%m.%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'thehexfiles.net'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.thehexfiles.net','thehexfiles.net']
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'cellspacing' : '4'}):
-            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                self.story.setMetadata('title',a.string)
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-			
-        words = info.find(text=re.compile('Words:')).split('|')[1].split(': ')[1]
-        self.story.setMetadata('numWords', words)
-		
-        comp = info.find('span', {'class' : 'completed'}).string.split(': ')[1]
-        if 'Yes' in comp:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-						
-        summary = info.find('td', {'class' : 'summary'})
-        self.setDescription(url,summary)
-		
-        rating=stripHTML(info.find('td', {'align' : 'left'})).split('(')[1].split(')')[0]
-        self.story.setMetadata('rating', rating)
-
-        labels = info.findAll('td', {'width' : '10%'})
-        values = info.findAll('td', {'width' : '40%'})
-        for i in range(0,len(labels)):
-            value = stripHTML(values[i])
-            label = stripHTML(labels[i])
-
-            if 'Genres' in label:
-                genres = value.split(', ')
-                for genre in genres:
-                    if genre != 'none':
-                        self.story.addToList('genre',genre)
-						
-
-            if 'Warnings' in label:
-                warnings = value.split(', ')
-                for warning in warnings:
-                    if warning != 'none':
-                        self.story.addToList('warnings',warning)
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr','img')) # otherwise soup eats the br/hr tags.
-
-        if None == soup:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        # Ugh.  chapter html doesn't haven't anything useful around it to demarcate.
-        for a in soup.findAll('table'):
-            a.extract()        
-
-        for a in soup.findAll('head'):
-            a.extract()
-
-        html = soup.find('html')
-        html.name='div'
-            
-        return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_thehookupzonenet.py

@@ -1,306 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# By virtue of being recent and requiring both is_adult and user/pass,
-# adapter_fanficcastletvnet.py is the best choice for learning to
-# write adapters--especially for sites that use the eFiction system.
-# Most sites that have ".../viewstory.php?sid=123" in the story URL
-# are eFiction.
-
-# For non-eFiction sites, it can be considerably more complex, but
-# this is still a good starting point.
-
-# In general an 'adapter' needs to do these five things:
-
-# - 'Register' correctly with the downloader
-# - Site Login (if needed)
-# - 'Are you adult?' check (if needed--some do one, some the other, some both)
-# - Grab the chapter list
-# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
-# - Grab the chapter texts
-
-# Search for XXX comments--that's where things are most likely to need changing.
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return TheHookupZoneNetAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/CriminalMinds/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','thupz') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'thehookupzone.net' # XXX
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/CriminalMinds/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if "Age Consent Required" in data: # XXX 
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/CriminalMinds/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/CriminalMinds/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_themasquenet.py

@@ -1,272 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return TheMasqueNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class TheMasqueNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        if self.parsedUrl.path.split('/',)[1] == 'wiktt':
-            self.story.addToList("category","Harry Potter")
-            self.section='/wiktt/efiction/'
-            self.dateformat = "%m/%d/%Y"
-        else:
-            self.story.addToList("category","Originals")
-            self.section='/efiction/'
-            self.dateformat = "%b %d, %Y"
-            
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + self.section + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','msq')
-        
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'themasque.net'
-        
-    def getSiteExampleURLs(self):
-        return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain())+"(/wiktt)?/efiction"+re.escape("/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain()  + self.section + 'user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.section + chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.text
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thepetulantpoetesscom.py

@@ -1,240 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ThePetulantPoetessComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ThePetulantPoetessComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','tpp')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y/%m/%d"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.thepetulantpoetess.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'You must be a member to read this story.' in data \
-                or "The Ministry of Magic does not have a record of that password. " in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "My Account Page" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        chapters=soup.find('select', {'name' : 'sid'})
-        if chapters == None:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters.findAll('option', value=re.compile(r"viewstory.php\?sid=\d+&i=1")):
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        # make sure that the story id is from the first chapter
-        self.story.setMetadata('storyId',self.chapterUrls[0][1].split('=')[1].split('&')[0])
-        
-        #locate the story on author's page
-        index = 1
-        found = 0
-        while found == 0:
-            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&page="+str(index)))
-        
-            for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
-                a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-                if a != None:
-                    self.story.setMetadata('title',a.text)                
-                    found = 1
-                    break
-            index=index+1
-            
-        # extract metadata
-        b=info.find('b')
-        b.find('a').extract()
-        self.story.setMetadata('rating', b.text.split('[')[1].split(']')[0])
-        
-        info = info.findNext('td', {'colspan' : '2'}) 
-        for label in info.findAll('b'):
-            value = label.nextSibling
-            
-            if "Category" in label.text:
-                for cat in info.findAll('a'):
-                    self.story.addToList('category',cat.string)
-            
-            if "Characters" in label.text:
-                for char in value.split(', '):
-                    self.story.addToList('characters',char)
-                    
-            if "Genres" in label.text:
-                for genre in value.split(', '):
-                    if "General" not in genre:
-                        self.story.addToList('genre',genre)
-                    
-            if "Warnings" in label.text:
-                for warning in value.split(', '):
-                    if "none" not in warning.lower():
-                        self.story.addToList('warnings',warning)
-                        
-                        
-        info = info.findNext('td', {'class' : 'tblborder'}) 
-        info.find('b').extract()
-        self.setDescription(url,info)
-        
-        info = info.findNext('td', {'class' : 'highlightcolor2'}) 
-        for label in info.findAll('b'):
-            value = label.nextSibling
-            
-            if "Completed" in label.text:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-                    
-            if 'Published' in label.text:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label.text:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                
-            if 'Word Count' in label.text:
-                self.story.setMetadata('numWords', value)
-            
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.findAll('table')[2].findAll('td')[1]
-        for a in div.findAll('div'):
-            a.extract()
-        div.name='div'
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -1,289 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return TheQuidditchPitchOrgAdapter # XXX
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        # XXX Most sites don't have the  part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','tqdpch') # XXX
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y" # XXX
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'thequidditchpitch.org' # XXX
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.thequidditchpitch.org','thequidditchpitch.org']
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only - Not suitable for readers under the age of legal consent in their country.' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4" # XXX
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-        if ("Not suitable for readers under the age of legal consent in their country." in data \
-                or "Not suitable for readers under 16 yrs. \r\nStories may contain violence, slight nudity, and/or sexual situations." in data ) \
-                and not (self.is_adult or self.getConfig("is_adult")): # XXX 
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        #print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        # span?  Really?  span?  Yeah... I don't think so.
-        div = soup.find('span', {'style' : 'font-size: 100%;'})
-        div.name='div'
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -1,259 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','twcs')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        self.dateformat = "%B %d, %Y"
-
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'www.thewriterscoffeeshop.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/library/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/library/viewstory.php?sid=")+r"\d+$"
-
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-          return True
-        else:
-          return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/library/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            addurl = "&ageconsent=ok&warning=3"
-        else:
-            addurl=""
-            
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        if "Age Consent Required" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):] 
-        
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/library/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/library/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-        
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    # poor HTML(unclosed <p> for one) can cause run on
-                    # over the next label.
-                    if '<span class="label">' in svalue:
-                        svalue = svalue[0:svalue.find('<span class="label">')]
-                        break
-                    else:
-                        value = value.nextSibling
-                self.setDescription(url,svalue)
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/library/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):]
-        
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
-
-def getClass():
-    return TheWritersCoffeeShopComSiteAdapter
-

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -1,280 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import time
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','tth')
-        self.dateformat = "%d %b %y"
-        self.is_adult=False
-        self.username = None
-        self.password = None
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-            # normalized story URL.
-            self._setURL("http://"+self.getSiteDomain()\
-                         +"/Story-"+self.story.getMetadata('storyId'))
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.tthfanfic.org'
-
-    def getSiteExampleURLs(self):
-        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm http://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
-
-    # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
-    # http://www.tthfanfic.org/Story-5583
-    # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
-    # http://www.tthfanfic.org/story.php?no=26093
-    def getSiteURLPattern(self):
-        return r"http://www.tthfanfic.org(/(T-\d+/)?Story-|/story.php\?no=)(?P<id>\d+)(-\d+)?(/.*)?$"
-
-    # tth won't send you future updates if you aren't 'caught up'
-    # on the story.  Login isn't required for F21, but logging in will
-    # mark stories you've downloaded as 'read' on tth.
-    def performLogin(self):
-        params = {}
-
-        if self.password:
-            params['urealname'] = self.username
-            params['password'] = self.password
-        else:
-            params['urealname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['loginsubmit'] = 'Login'
-
-        if not params['password']:
-            return
-        
-        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['urealname']))
-
-        ## need to pull empty login page first to get ctkn and
-        ## password name, which are BUSs
-# <form method='post' action='/login.php' accept-charset="utf-8">
-# <input type='hidden' name='ctkn' value='4bdf761f5bea06bf4477072afcbd0f8d721d1a4f989c09945a9e87afb7a66de1'/>
-# <input type='text' id='urealname' name='urealname' value=''/>
-# <input type='password' id='password' name='6bb3fcd148d148629223690bf19733b8'/>
-# <input type='submit' value='Login' name='loginsubmit'/>
-        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
-        params['ctkn']=soup.find('input', {'name':'ctkn'})['value']
-        params[soup.find('input', {'id':'password'})['name']] = params['password']
-        
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Stories Published" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    def extractChapterUrlsAndMetadata(self):
-        # fetch the chapter.  From that we will get almost all the
-        # metadata and chapter list
-
-        url=self.url
-        logger.debug("URL: "+url)
-
-        # tth won't send you future updates if you aren't 'caught up'
-        # on the story.  Login isn't required for F21, but logging in will
-        # mark stories you've downloaded as 'read' on tth.
-        self.performLogin()
-        
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        try:
-            data = self._fetchUrl(url)
-            soup = bs.BeautifulSoup(data)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(url)
-            else:
-                raise e
-            
-        descurl = url
-            
-        if "<h2>Story Not Found</h2>" in data:
-            raise exceptions.StoryDoesNotExist(url)
-        
-        if "NOTE: This story is rated FR21 which is above your chosen filter level." in data:
-            if self.is_adult or self.getConfig("is_adult"):
-                form = soup.find('form', {'id':'sitemaxratingform'})
-                params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
-                        'sitemaxrating':'5'}
-                logger.info("Attempting to get rating cookie for %s" % url)
-                data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
-                # refetch story page.
-                data = self._fetchUrl(url)
-                soup = bs.BeautifulSoup(data)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        # http://www.tthfanfic.org/AuthorStories-3449/Greywizard.htm
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/AuthorStories-\d+"))
-        self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',stripHTML(a))
-        authorurl = 'http://'+self.host+a['href']
-
-        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
-        if ainfo != None: # indicates multiple authors/contributors.
-            try:
-                # going to pull part of the meta data from author list page.
-                infourl = 'http://'+self.host+ainfo['href']
-                logger.debug("**StoryInfo** URL: "+infourl)
-                infodata = self._fetchUrl(infourl)
-                infosoup = bs.BeautifulSoup(infodata)
-
-                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
-                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
-                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
-                    self.story.addToList('author',stripHTML(a))                
-                                            
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(url)
-                else:
-                    raise e
-    
-        try:
-            # going to pull part of the meta data from *primary* author list page.
-            logger.debug("**AUTHOR** URL: "+authorurl)
-            authordata = self._fetchUrl(authorurl)
-            descurl=authorurl
-            authorsoup = bs.BeautifulSoup(authordata)
-            # author can have several pages, scan until we find it.
-            while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
-                nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
-                logger.debug("**AUTHOR** nextpage URL: "+nextpage)
-                authordata = self._fetchUrl(nextpage)
-                descurl=nextpage
-                authorsoup = bs.BeautifulSoup(authordata)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(url)
-            else:
-                raise e
-
-        storydiv = authorsoup.find('div', {'id':'st'+self.story.getMetadata('storyId'), 'class':re.compile(r"storylistitem")})
-        self.setDescription(descurl,storydiv.find('div',{'class':'storydesc'}))
-        #self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
-        self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))
-
-        verticaltable = soup.find('table', {'class':'verticaltable'})
-
-        BtVS = True
-        BtVSNonX = False
-        for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
-            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
-                self.story.addToList('category',cat.string)
-            else:
-                if 'Non-BtVS' in cat.string:
-                    BtVS = False
-                if 'BtVS/AtS Non-Crossover' == cat.string:
-                    BtVSNonX = True
-
-        verticaltabletds = verticaltable.findAll('td')
-        self.story.setMetadata('rating', verticaltabletds[2].string)
-        self.story.setMetadata('numWords', verticaltabletds[4].string)
-        
-        # Complete--if completed.
-        if 'Yes' in verticaltabletds[10].string:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-            
-        self.story.setMetadata('datePublished',makeDate(stripHTML(verticaltabletds[8].string), self.dateformat))
-        self.story.setMetadata('dateUpdated',makeDate(stripHTML(verticaltabletds[9].string), self.dateformat))
-
-        for icon in storydiv.find('span',{'class':'storyicons'}).findAll('img'):
-            if( icon['title'] not in ['Non-Crossover'] ) :
-                self.story.addToList('genre',icon['title'])
-            else:
-                if not BtVSNonX:
-                    BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover
-
-        #print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
-        if BtVS:
-            self.story.addToList('category','Buffy: The Vampire Slayer')
-            
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapnav' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = "http://"+self.host+o['value']
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(o),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        pseries = soup.find('p', {'style':'margin-top:0px'})
-        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
-                     pseries.text)
-        if m:
-            self.setSeries(m.group('series'),m.group('num'))
-        
-        return
-
-
-    def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
-
-        div = soup.find('div', {'id' : 'storyinnerbody'})
-        
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        # strip out included chapter title, if present, to avoid doubling up.
-        try:
-            div.find('h3').extract()
-        except:
-            pass
-        return self.utf8FromSoup(url,div)
-
-def getClass():
-    return TwistingTheHellmouthSiteAdapter
-

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -1,186 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return TwilightArchivesComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class TwilightArchivesComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL. http://www.twilightarchives.com/read/9353
-        self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','twa')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.twilightarchives.com'
-       
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/read/1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://" + self.getSiteDomain()+"/read/")+r"\d+(/d+)?$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
-        
-        # Find the chapters:
-        chapters=soup.find('ol', {'class' : 'chapters'})
-        if chapters != None:
-            for chapter in chapters.findAll('a', href=re.compile(r'/read/'+self.story.getMetadata('storyId')+"/\d+$")):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-                
-# rated, genre, warnings, seires
-
-        summary = soup.find('p', {'class' : 'images'})
-        self.setDescription(url,summary)
-
-        for c in soup.findAll('h2', {'class' : 'title'}):
-            div = c.nextSibling.nextSibling
-            
-            if 'Information' in c.text:
-                for dt in div.findAll('dt'):
-                    dd=dt.nextSibling.nextSibling
-                    
-                    if 'Author' in dt.text:
-                        a=dd.find('a')
-                        self.story.setMetadata('authorId',a['href'].split('/')[2])
-                        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-                        self.story.setMetadata('author',a.text)
-                        
-                    if 'Words' in dt.text:
-                        self.story.setMetadata('numWords', dd.text)
-                        
-                    if 'Published' in dt.text:
-                        self.story.setMetadata('datePublished', makeDate(stripHTML(dd.text), self.dateformat))
-            
-                    if 'Updated' in dt.text:
-                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(dd.text), self.dateformat))
-                    
-                    if 'Status' in dt.text:
-                        if 'Complete' in dd.text:
-                            self.story.setMetadata('status', 'Completed')
-                        else:
-                            self.story.setMetadata('status', 'In-Progress')
-
-            if 'Categories' in c.text:
-                for a in div.findAll('a'):
-                    self.story.addToList('category',a.text)
-                    
-            if 'Characters' in c.text:
-                for a in div.findAll('a'):
-                    self.story.addToList('category',a.text)
-
-            if 'Series' in c.text:
-                a=div.find('a')
-                series_name = a.text
-                series_url = 'http://'+self.host+a['href']
-                
-                seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-                storyas = seriessoup.find('tbody').findAll('a', href=re.compile(r'^/read/\d+$'))
-                i=1
-                for a in storyas:
-                    if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-                    
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        a=asoup.find('tbody').find('a', href=re.compile(r'^/read/'+self.story.getMetadata('storyId')))
-        self.story.setMetadata('rating',a.parent.nextSibling.nextSibling.text)
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'class' : 'size images medium'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_twilightednet.py

@@ -1,250 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class TwilightedNetSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','tw')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'www.twilighted.net'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.twilighted.net','twilighted.net']
-
-    def getSiteExampleURLs(self):
-        return "http://www.twilighted.net/viewstory.php?sid=1234 http://twilighted.net/viewstory.php?sid=5678"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?"+re.escape("twilighted.net/viewstory.php?sid=")+r"\d+$"
-
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-          return True
-        else:
-          return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    def extractChapterUrlsAndMetadata(self):
-
-        url = self.url+'&index=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        # twilighted isn't writing <body> ??? wtf?
-        data = "<html><body>"+data[data.index("</head>"):] 
-        
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-        
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## twilighted.net doesn't use genre.
-            # if 'Genre' in label:
-            #     genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
-            #     genrestext = [genre.string for genre in genres]
-            #     self.genre = ', '.join(genrestext)
-            #     for genre in genrestext:
-            #         self.story.addToList('genre',genre.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%B %d, %Y"))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-            
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-        
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        # twilighted isn't writing <body> ??? wtf?
-        data = "<html><body>"+data[data.index("</head>"):] 
-        
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
-
-def getClass():
-    return TwilightedNetSiteAdapter
-

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -1,276 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class TwiwriteNetSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','twrt')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.is_adult = False
-        self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
-        self.password = ""
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-            
-    @staticmethod
-    def getSiteDomain():
-        return 'www.twiwrite.net'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.twiwrite.net','twiwrite.net']
-
-    def getSiteExampleURLs(self):
-        return "http://www.twiwrite.net/viewstory.php?sid=1234 http://twiwrite.net/viewstory.php?sid=5678"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?"+re.escape("twiwrite.net/viewstory.php?sid=")+r"\d+$"
-
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-          return True
-        else:
-          return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.info("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=1" # XXX
-        else:
-            addurl=""
-            
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        if "Contains Explicit Content for mature adults only! May contain graphic violence, mature sexual situations, and explicit language. Read with caution." in data:
-            raise exceptions.AdultCheckRequired(self.url)
-        
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):] 
-        
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        ## <meta name='description' content='&lt;p&gt;Description&lt;/p&gt; ...' >
-        ## Summary, strangely, is in the content attr of a <meta name='description'> tag
-        ## which is escaped HTML.  Unfortunately, we can't use it because they don't
-        ## escape (') chars in the desc, breakin the tag.
-        #meta_desc = soup.find('meta',{'name':'description'})
-        #metasoup = bs.BeautifulStoneSoup(meta_desc['content'])
-        #self.story.setMetadata('description',stripHTML(metasoup))
-
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-        
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=8'))
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warning',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%B %d, %Y"))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        # problems with some stories, but only in calibre.  I suspect
-        # issues with different SGML parsers in python.  This is a
-        # nasty hack, but it works.
-        data = data[data.index("<body"):]
-        
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
-
-def getClass():
-    return TwiwriteNetSiteAdapter
-

fanficdownloader/adapters/adapter_walkingtheplankorg.py

@@ -1,229 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return WalkingThePlankOrgAdapter
-
-class WalkingThePlankOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','wtp')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.walkingtheplank.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-    
-        if "By clicking this link, you acknowledge" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
-        
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-				
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/archive/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_whoficcom.py

@@ -1,233 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-class WhoficComSiteAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-        self.story.setMetadata('siteabbrev','whof')
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        
-    @staticmethod
-    def getSiteDomain():
-        return 'www.whofic.com'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+"\d+$"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-        # fetch the first chapter.  From that we will:
-        # - determine title, authorname, authorid
-        # - get chapter list, if not one-shot.
-
-        url = self.url+'&chapter=1'
-        logger.debug("URL: "+url)
-        
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        try:
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        # pull title(title) and author from the HTML title.
-        title = soup.find('title').string
-        logger.debug('Title: %s' % title)
-        title = title.split('::')[1].strip()
-        self.story.setMetadata('title',title.split(' by ')[0].strip())
-        self.story.setMetadata('author',title.split(' by ')[1].strip())
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapter' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-    	   allOptions = select.findAll('option')
-    	   for o in allOptions:
-    	     url = self.url + "&chapter=%s" % o['value']
-             # just in case there's tags, like <i> in chapter titles.
-    	     title = "%s" % o
-             title = re.sub(r'<[^>]+>','',title)
-    	     self.chapterUrls.append((title,url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        ## Whofic.com puts none of the other meta data in the chapters
-        ## or even the story chapter index page.  Need to scrape the
-        ## author page to find it.
-
-        # <table width="100%" bordercolor="#333399" border="0" cellspacing="0" cellpadding="2"><tr><td>
-        # <b><a href="viewstory.php?sid=38220">Accompaniment 2</a></b> by <a href="viewuser.php?uid=12412">clandestinemiscreant</a>  [<a href="reviews.php?sid=38220">Reviews</a> - <a href="reviews.php?sid=38220">0</a>] <br>
-        # This is a series of short stories written as an accompaniment to Season 2, Season 28 for us oldies, and each is unrelated except for that one factor. Each story is canon, in that it does not change established events at time of airing, based on things mentioned and/or implied and missing or deleted scenes that were not seen in the final aired episodes.<br>
-        # <font size="-1"><b><a href="categories.php?catid=15">Tenth Doctor</a></b> - All Ages - None - Humor, Hurt/Comfort, Romance<br>
-        # <i>Characters:</i> Rose Tyler<br>
-        # <i>Series:</i> None<br>
-        # <i>Published:</i> 2010.08.15 - <i>Updated:</i> 2010.08.16 - <i>Chapters:</i> 4 - <i>Completed:</i> Yes - <i>Word Count:</i> 4890 </font>
-        # </td></tr></table>
-             
-        logger.debug("Author URL: "+self.story.getMetadata('authorUrl'))
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(self.story.getMetadata('authorUrl')),
-                                     selfClosingTags=('br')) # normalize <br> tags to <br />
-        # find this story in the list, parse it's metadata based on
-        # lots of assumptions about the html, since there's little
-        # tagging.
-        # Found a story once that had the story URL in the desc for a
-        # series on the same author's page.  Now using the reviews
-        # link instead to find the appropriate metadata.
-        a = soup.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
-        metadata = a.findParent('td')
-        metadatachunks = self.utf8FromSoup(None,metadata).split('<br />')
-        # process metadata for this story.
-        self.setDescription(url,metadatachunks[1])
-        #self.story.setMetadata('description', metadatachunks[1])
-
-        # First line of the stuff with ' - ' separators
-        moremeta = metadatachunks[2]
-        moremeta = re.sub(r'<[^>]+>','',moremeta) # strip tags.
-        
-        moremetaparts = moremeta.split(' - ')
-        
-        # first part is category--whofic.com has categories
-        # Doctor One-11, Torchwood, etc.  We're going to
-        # prepend any with 'Doctor' or 'Era' (Multi-Era, Other
-        # Era) as 'Doctor Who'.
-        #
-        # Also push each in as 'extra tags'.
-        category = moremetaparts[0]
-        if 'Doctor' in category or 'Era' in category :
-            self.story.addToList('category','Doctor Who')
-            
-        for cat in category.split(', '):
-            self.story.addToList('category',cat)
-
-        # next in that line is age rating.
-        self.story.setMetadata('rating',moremetaparts[1])
-
-        # after that is a possible list fo specific warnings,
-        # Explicit Violence, Swearing, etc
-        if "None" not in moremetaparts[2]:
-            for warn in moremetaparts[2].split(', '):
-                self.story.addToList('warnings',warn)
-
-        # then genre.  It's another comma list.  All together
-        # in genre, plus each in extra tags.
-        genre=moremetaparts[3]
-        for g in genre.split(r', '):
-            self.story.addToList('genre',g)
-
-        # line 3 is characters.
-        chars = metadatachunks[3]
-        charsearch="<i>Characters:</i>"
-        if charsearch in chars:
-            chars = chars[metadatachunks[3].index(charsearch)+len(charsearch):]
-            for c in chars.split(','):
-                if c.strip() != u'None':
-                    self.story.addToList('characters',c)
-            
-        # the next line is stuff with ' - ' separators *and* names--with tags.
-        moremeta = metadatachunks[5]
-        moremeta = re.sub(r'<[^>]+>','',moremeta) # strip tags.
-        
-        moremetaparts = moremeta.split(' - ')
-
-        for part in moremetaparts:
-            (name,value) = part.split(': ')
-            name=name.strip()
-            value=value.strip()
-            if name == 'Published':
-                self.story.setMetadata('datePublished', makeDate(value, '%Y.%m.%d'))
-            if name == 'Updated':
-                self.story.setMetadata('dateUpdated', makeDate(value, '%Y.%m.%d'))
-            if name == 'Completed':
-                if value == 'Yes':
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-            if name == 'Word Count':
-                self.story.setMetadata('numWords', value)
-
-        try:
-            # Find Series name from series URL.
-            a = metadata.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-
-        # hardly a great identifier, I know, but whofic really doesn't
-        # give us anything better to work with.
-        span = soup.find('span', {'style' : 'font-size: 100%;'})
-
-        if None == span:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
-
-def getClass():
-    return WhoficComSiteAdapter
-

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -1,300 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return WizardTalesNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class WizardTalesNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','wzt')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.wizardtales.net'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title
-        pt = soup.find('div', {'id' : 'pagetitle'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
-        
-        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        rating=pt.text.split('[')[1].split(']')[0]
-        self.story.setMetadata('rating', rating)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        content=soup.find('div',{'class' : 'content'})
-        
-        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
-            self.story.addToList('genre',genre.string)
-
-        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
-            self.story.addToList('warnings',warning.string)
-        
-        labels = content.findAll('b')
-        
-        value = labels[0].previousSibling
-        svalue = ""
-        while value != None:
-            val = value
-            value = value.previousSibling
-        while "Categories" not in val:
-            svalue += str(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
-        
-        
-        
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', stripHTML(value).split(': ')[1].split(';')[0])
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value).split(': ')[1].split(';')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value).split(': ')[1], self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.findAll('div', {'id' : 'story'})[1]
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)