commit 59ae4fe374ba2232f39cda08f530ab7d41d7ebd7 Author: Jim Miller Date: Sun Dec 28 17:21:39 2014 -0600 Add universe & crossoverfandom site-specific to ksarchive, clean up ini files. diff --git a/allrecent.html b/allrecent.html new file mode 100644 index 0000000..477b17b --- /dev/null +++ b/allrecent.html @@ -0,0 +1,78 @@ + + + + + FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML) + + + + +
+

+ FanFictionDownLoader +

+ + + + + {{yourfile}} + + +
+ {% for fic in fics %} +

+ {{ fic.title }} + by {{ fic.author }} Download Count: {{ fic.count }}
+ Word Count: {{ fic.numWords }} Chapter Count: {{ fic.numChapters }}
+ {% if fic.category %} Categories: {{ fic.category }}
{% endif %} + {% if fic.genre %} Genres: {{ fic.genre }}
{% endif %} + {% if fic.language %} Language: {{ fic.language }}
{% endif %} + {% if fic.series %} Series: {{ fic.series }}
{% endif %} + {% if fic.characters %} Characters: {{ fic.characters }}
{% endif %} + {% if fic.status %} Status: {{ fic.status }}
{% endif %} + {% if fic.datePublished %} Published: {{ fic.datePublished }}
{% endif %} + {% if fic.dateUpdated %} Last Updated: {{ fic.dateUpdated }}
{% endif %} + {% if fic.dateCreated %} Last Downloaded: {{ fic.dateCreated }}
{% endif %} + {% if fic.rating %} Rating: {{ fic.rating }}
{% endif %} + {% if fic.warnings %} Warnings: {{ fic.warnings }}
{% endif %} + {% if fic.description %} Summary: {{ fic.description }}
{% endif %} +

+ {% endfor %} +
+ + + + +
+ + diff --git a/app.yaml b/app.yaml new file mode 100644 index 0000000..d04aac4 --- /dev/null +++ b/app.yaml @@ -0,0 +1,46 @@ +# ffd-retief-hrd fanfictiondownloader +application: fanfictiondownloader +version: 2-1-01 +runtime: python27 +api_version: 1 +threadsafe: true + +handlers: + +- url: /r3m0v3r.* + script: utils.remover.app + login: admin + +- url: /tally.* + script: utils.tally.app + login: admin + +- url: /fdownloadtask + script: main.app + login: admin + +- url: /css + static_dir: css + +- url: /js + static_dir: js + +- url: /static + static_dir: static + +- url: /favicon\.ico + static_files: static/favicon.ico + upload: static/favicon\.ico + +- url: /.* + script: main.app + +#builtins: +#- datastore_admin: on + +libraries: +- name: django + version: "1.2" + +- name: PIL + version: "1.1.7" diff --git a/bs4/COPYING.txt b/bs4/COPYING.txt new file mode 100644 index 0000000..d668d13 --- /dev/null +++ b/bs4/COPYING.txt @@ -0,0 +1,26 @@ +Beautiful Soup is made available under the MIT license: + + Copyright (c) 2004-2012 Leonard Richardson + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE, DAMMIT. + +Beautiful Soup incorporates code from the html5lib library, which is +also made available under the MIT license. diff --git a/bs4/__init__.py b/bs4/__init__.py new file mode 100644 index 0000000..7ba3426 --- /dev/null +++ b/bs4/__init__.py @@ -0,0 +1,406 @@ +"""Beautiful Soup +Elixir and Tonic +"The Screen-Scraper's Friend" +http://www.crummy.com/software/BeautifulSoup/ + +Beautiful Soup uses a pluggable XML or HTML parser to parse a +(possibly invalid) document into a tree representation. Beautiful Soup +provides provides methods and Pythonic idioms that make it easy to +navigate, search, and modify the parse tree. + +Beautiful Soup works with Python 2.6 and up. It works better if lxml +and/or html5lib is installed. + +For more than you ever wanted to know about Beautiful Soup, see the +documentation: +http://www.crummy.com/software/BeautifulSoup/bs4/doc/ +""" + +__author__ = "Leonard Richardson (leonardr@segfault.org)" +__version__ = "4.3.2" +__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson" +__license__ = "MIT" + +__all__ = ['BeautifulSoup'] + +import os +import re +import warnings + +from .builder import builder_registry, ParserRejectedMarkup +from .dammit import UnicodeDammit +from .element import ( + CData, + Comment, + DEFAULT_OUTPUT_ENCODING, + Declaration, + Doctype, + NavigableString, + PageElement, + ProcessingInstruction, + ResultSet, + SoupStrainer, + Tag, + ) + +# The very first thing we do is give a useful error if someone is +# running this code under Python 3 without converting it. +syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).' + +class BeautifulSoup(Tag): + """ + This class defines the basic interface called by the tree builders. + + These methods will be called by the parser: + reset() + feed(markup) + + The tree builder may call these methods from its feed() implementation: + handle_starttag(name, attrs) # See note about return value + handle_endtag(name) + handle_data(data) # Appends to the current data node + endData(containerClass=NavigableString) # Ends the current data node + + No matter how complicated the underlying parser is, you should be + able to build a tree using 'start tag' events, 'end tag' events, + 'data' events, and "done with data" events. + + If you encounter an empty-element tag (aka a self-closing tag, + like HTML's
tag), call handle_starttag and then + handle_endtag. + """ + ROOT_TAG_NAME = u'[document]' + + # If the end-user gives no indication which tree builder they + # want, look for one with these features. + DEFAULT_BUILDER_FEATURES = ['html', 'fast'] + + ASCII_SPACES = '\x20\x0a\x09\x0c\x0d' + + def __init__(self, markup="", features=None, builder=None, + parse_only=None, from_encoding=None, **kwargs): + """The Soup object is initialized as the 'root tag', and the + provided markup (which can be a string or a file-like object) + is fed into the underlying parser.""" + + if 'convertEntities' in kwargs: + warnings.warn( + "BS4 does not respect the convertEntities argument to the " + "BeautifulSoup constructor. Entities are always converted " + "to Unicode characters.") + + if 'markupMassage' in kwargs: + del kwargs['markupMassage'] + warnings.warn( + "BS4 does not respect the markupMassage argument to the " + "BeautifulSoup constructor. The tree builder is responsible " + "for any necessary markup massage.") + + if 'smartQuotesTo' in kwargs: + del kwargs['smartQuotesTo'] + warnings.warn( + "BS4 does not respect the smartQuotesTo argument to the " + "BeautifulSoup constructor. Smart quotes are always converted " + "to Unicode characters.") + + if 'selfClosingTags' in kwargs: + del kwargs['selfClosingTags'] + warnings.warn( + "BS4 does not respect the selfClosingTags argument to the " + "BeautifulSoup constructor. The tree builder is responsible " + "for understanding self-closing tags.") + + if 'isHTML' in kwargs: + del kwargs['isHTML'] + warnings.warn( + "BS4 does not respect the isHTML argument to the " + "BeautifulSoup constructor. You can pass in features='html' " + "or features='xml' to get a builder capable of handling " + "one or the other.") + + def deprecated_argument(old_name, new_name): + if old_name in kwargs: + warnings.warn( + 'The "%s" argument to the BeautifulSoup constructor ' + 'has been renamed to "%s."' % (old_name, new_name)) + value = kwargs[old_name] + del kwargs[old_name] + return value + return None + + parse_only = parse_only or deprecated_argument( + "parseOnlyThese", "parse_only") + + from_encoding = from_encoding or deprecated_argument( + "fromEncoding", "from_encoding") + + if len(kwargs) > 0: + arg = kwargs.keys().pop() + raise TypeError( + "__init__() got an unexpected keyword argument '%s'" % arg) + + if builder is None: + if isinstance(features, basestring): + features = [features] + if features is None or len(features) == 0: + features = self.DEFAULT_BUILDER_FEATURES + builder_class = builder_registry.lookup(*features) + if builder_class is None: + raise FeatureNotFound( + "Couldn't find a tree builder with the features you " + "requested: %s. Do you need to install a parser library?" + % ",".join(features)) + builder = builder_class() + self.builder = builder + self.is_xml = builder.is_xml + self.builder.soup = self + + self.parse_only = parse_only + + if hasattr(markup, 'read'): # It's a file-type object. + markup = markup.read() + elif len(markup) <= 256: + # Print out warnings for a couple beginner problems + # involving passing non-markup to Beautiful Soup. + # Beautiful Soup will still parse the input as markup, + # just in case that's what the user really wants. + if (isinstance(markup, unicode) + and not os.path.supports_unicode_filenames): + possible_filename = markup.encode("utf8") + else: + possible_filename = markup + is_file = False + try: + is_file = os.path.exists(possible_filename) + except Exception, e: + # This is almost certainly a problem involving + # characters not valid in filenames on this + # system. Just let it go. + pass + if is_file: + warnings.warn( + '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup) + if markup[:5] == "http:" or markup[:6] == "https:": + # TODO: This is ugly but I couldn't get it to work in + # Python 3 otherwise. + if ((isinstance(markup, bytes) and not b' ' in markup) + or (isinstance(markup, unicode) and not u' ' in markup)): + warnings.warn( + '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup) + + for (self.markup, self.original_encoding, self.declared_html_encoding, + self.contains_replacement_characters) in ( + self.builder.prepare_markup(markup, from_encoding)): + self.reset() + try: + self._feed() + break + except ParserRejectedMarkup: + pass + + # Clear out the markup and remove the builder's circular + # reference to this object. + self.markup = None + self.builder.soup = None + + def _feed(self): + # Convert the document to Unicode. + self.builder.reset() + + self.builder.feed(self.markup) + # Close out any unfinished strings and close all the open tags. + self.endData() + while self.currentTag.name != self.ROOT_TAG_NAME: + self.popTag() + + def reset(self): + Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME) + self.hidden = 1 + self.builder.reset() + self.current_data = [] + self.currentTag = None + self.tagStack = [] + self.preserve_whitespace_tag_stack = [] + self.pushTag(self) + + def new_tag(self, name, namespace=None, nsprefix=None, **attrs): + """Create a new tag associated with this soup.""" + return Tag(None, self.builder, name, namespace, nsprefix, attrs) + + def new_string(self, s, subclass=NavigableString): + """Create a new NavigableString associated with this soup.""" + navigable = subclass(s) + navigable.setup() + return navigable + + def insert_before(self, successor): + raise NotImplementedError("BeautifulSoup objects don't support insert_before().") + + def insert_after(self, successor): + raise NotImplementedError("BeautifulSoup objects don't support insert_after().") + + def popTag(self): + tag = self.tagStack.pop() + if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]: + self.preserve_whitespace_tag_stack.pop() + #print "Pop", tag.name + if self.tagStack: + self.currentTag = self.tagStack[-1] + return self.currentTag + + def pushTag(self, tag): + #print "Push", tag.name + if self.currentTag: + self.currentTag.contents.append(tag) + self.tagStack.append(tag) + self.currentTag = self.tagStack[-1] + if tag.name in self.builder.preserve_whitespace_tags: + self.preserve_whitespace_tag_stack.append(tag) + + def endData(self, containerClass=NavigableString): + if self.current_data: + current_data = u''.join(self.current_data) + # If whitespace is not preserved, and this string contains + # nothing but ASCII spaces, replace it with a single space + # or newline. + if not self.preserve_whitespace_tag_stack: + strippable = True + for i in current_data: + if i not in self.ASCII_SPACES: + strippable = False + break + if strippable: + if '\n' in current_data: + current_data = '\n' + else: + current_data = ' ' + + # Reset the data collector. + self.current_data = [] + + # Should we add this string to the tree at all? + if self.parse_only and len(self.tagStack) <= 1 and \ + (not self.parse_only.text or \ + not self.parse_only.search(current_data)): + return + + o = containerClass(current_data) + self.object_was_parsed(o) + + def object_was_parsed(self, o, parent=None, most_recent_element=None): + """Add an object to the parse tree.""" + parent = parent or self.currentTag + most_recent_element = most_recent_element or self._most_recent_element + o.setup(parent, most_recent_element) + + if most_recent_element is not None: + most_recent_element.next_element = o + self._most_recent_element = o + parent.contents.append(o) + + def _popToTag(self, name, nsprefix=None, inclusivePop=True): + """Pops the tag stack up to and including the most recent + instance of the given tag. If inclusivePop is false, pops the tag + stack up to but *not* including the most recent instqance of + the given tag.""" + #print "Popping to %s" % name + if name == self.ROOT_TAG_NAME: + # The BeautifulSoup object itself can never be popped. + return + + most_recently_popped = None + + stack_size = len(self.tagStack) + for i in range(stack_size - 1, 0, -1): + t = self.tagStack[i] + if (name == t.name and nsprefix == t.prefix): + if inclusivePop: + most_recently_popped = self.popTag() + break + most_recently_popped = self.popTag() + + return most_recently_popped + + def handle_starttag(self, name, namespace, nsprefix, attrs): + """Push a start tag on to the stack. + + If this method returns None, the tag was rejected by the + SoupStrainer. You should proceed as if the tag had not occured + in the document. For instance, if this was a self-closing tag, + don't call handle_endtag. + """ + + # print "Start tag %s: %s" % (name, attrs) + self.endData() + + if (self.parse_only and len(self.tagStack) <= 1 + and (self.parse_only.text + or not self.parse_only.search_tag(name, attrs))): + return None + + tag = Tag(self, self.builder, name, namespace, nsprefix, attrs, + self.currentTag, self._most_recent_element) + if tag is None: + return tag + if self._most_recent_element: + self._most_recent_element.next_element = tag + self._most_recent_element = tag + self.pushTag(tag) + return tag + + def handle_endtag(self, name, nsprefix=None): + #print "End tag: " + name + self.endData() + self._popToTag(name, nsprefix) + + def handle_data(self, data): + self.current_data.append(data) + + def decode(self, pretty_print=False, + eventual_encoding=DEFAULT_OUTPUT_ENCODING, + formatter="minimal"): + """Returns a string or Unicode representation of this document. + To get Unicode, pass None for encoding.""" + + if self.is_xml: + # Print the XML declaration + encoding_part = '' + if eventual_encoding != None: + encoding_part = ' encoding="%s"' % eventual_encoding + prefix = u'\n' % encoding_part + else: + prefix = u'' + if not pretty_print: + indent_level = None + else: + indent_level = 0 + return prefix + super(BeautifulSoup, self).decode( + indent_level, eventual_encoding, formatter) + +# Alias to make it easier to type import: 'from bs4 import _soup' +_s = BeautifulSoup +_soup = BeautifulSoup + +class BeautifulStoneSoup(BeautifulSoup): + """Deprecated interface to an XML parser.""" + + def __init__(self, *args, **kwargs): + kwargs['features'] = 'xml' + warnings.warn( + 'The BeautifulStoneSoup class is deprecated. Instead of using ' + 'it, pass features="xml" into the BeautifulSoup constructor.') + super(BeautifulStoneSoup, self).__init__(*args, **kwargs) + + +class StopParsing(Exception): + pass + +class FeatureNotFound(ValueError): + pass + + +#By default, act as an HTML pretty-printer. +if __name__ == '__main__': + import sys + soup = BeautifulSoup(sys.stdin) + print soup.prettify() diff --git a/bs4/builder/__init__.py b/bs4/builder/__init__.py new file mode 100644 index 0000000..740f5f2 --- /dev/null +++ b/bs4/builder/__init__.py @@ -0,0 +1,321 @@ +from collections import defaultdict +import itertools +import sys +from bs4.element import ( + CharsetMetaAttributeValue, + ContentMetaAttributeValue, + whitespace_re + ) + +__all__ = [ + 'HTMLTreeBuilder', + 'SAXTreeBuilder', + 'TreeBuilder', + 'TreeBuilderRegistry', + ] + +# Some useful features for a TreeBuilder to have. +FAST = 'fast' +PERMISSIVE = 'permissive' +STRICT = 'strict' +XML = 'xml' +HTML = 'html' +HTML_5 = 'html5' + + +class TreeBuilderRegistry(object): + + def __init__(self): + self.builders_for_feature = defaultdict(list) + self.builders = [] + + def register(self, treebuilder_class): + """Register a treebuilder based on its advertised features.""" + for feature in treebuilder_class.features: + self.builders_for_feature[feature].insert(0, treebuilder_class) + self.builders.insert(0, treebuilder_class) + + def lookup(self, *features): + if len(self.builders) == 0: + # There are no builders at all. + return None + + if len(features) == 0: + # They didn't ask for any features. Give them the most + # recently registered builder. + return self.builders[0] + + # Go down the list of features in order, and eliminate any builders + # that don't match every feature. + features = list(features) + features.reverse() + candidates = None + candidate_set = None + while len(features) > 0: + feature = features.pop() + we_have_the_feature = self.builders_for_feature.get(feature, []) + if len(we_have_the_feature) > 0: + if candidates is None: + candidates = we_have_the_feature + candidate_set = set(candidates) + else: + # Eliminate any candidates that don't have this feature. + candidate_set = candidate_set.intersection( + set(we_have_the_feature)) + + # The only valid candidates are the ones in candidate_set. + # Go through the original list of candidates and pick the first one + # that's in candidate_set. + if candidate_set is None: + return None + for candidate in candidates: + if candidate in candidate_set: + return candidate + return None + +# The BeautifulSoup class will take feature lists from developers and use them +# to look up builders in this registry. +builder_registry = TreeBuilderRegistry() + +class TreeBuilder(object): + """Turn a document into a Beautiful Soup object tree.""" + + features = [] + + is_xml = False + preserve_whitespace_tags = set() + empty_element_tags = None # A tag will be considered an empty-element + # tag when and only when it has no contents. + + # A value for these tag/attribute combinations is a space- or + # comma-separated list of CDATA, rather than a single CDATA. + cdata_list_attributes = {} + + + def __init__(self): + self.soup = None + + def reset(self): + pass + + def can_be_empty_element(self, tag_name): + """Might a tag with this name be an empty-element tag? + + The final markup may or may not actually present this tag as + self-closing. + + For instance: an HTMLBuilder does not consider a

tag to be + an empty-element tag (it's not in + HTMLBuilder.empty_element_tags). This means an empty

tag + will be presented as "

", not "

". + + The default implementation has no opinion about which tags are + empty-element tags, so a tag will be presented as an + empty-element tag if and only if it has no contents. + "" will become "", and "bar" will + be left alone. + """ + if self.empty_element_tags is None: + return True + return tag_name in self.empty_element_tags + + def feed(self, markup): + raise NotImplementedError() + + def prepare_markup(self, markup, user_specified_encoding=None, + document_declared_encoding=None): + return markup, None, None, False + + def test_fragment_to_document(self, fragment): + """Wrap an HTML fragment to make it look like a document. + + Different parsers do this differently. For instance, lxml + introduces an empty tag, and html5lib + doesn't. Abstracting this away lets us write simple tests + which run HTML fragments through the parser and compare the + results against other HTML fragments. + + This method should not be used outside of tests. + """ + return fragment + + def set_up_substitutions(self, tag): + return False + + def _replace_cdata_list_attribute_values(self, tag_name, attrs): + """Replaces class="foo bar" with class=["foo", "bar"] + + Modifies its input in place. + """ + if not attrs: + return attrs + if self.cdata_list_attributes: + universal = self.cdata_list_attributes.get('*', []) + tag_specific = self.cdata_list_attributes.get( + tag_name.lower(), None) + for attr in attrs.keys(): + if attr in universal or (tag_specific and attr in tag_specific): + # We have a "class"-type attribute whose string + # value is a whitespace-separated list of + # values. Split it into a list. + value = attrs[attr] + if isinstance(value, basestring): + values = whitespace_re.split(value) + else: + # html5lib sometimes calls setAttributes twice + # for the same tag when rearranging the parse + # tree. On the second call the attribute value + # here is already a list. If this happens, + # leave the value alone rather than trying to + # split it again. + values = value + attrs[attr] = values + return attrs + +class SAXTreeBuilder(TreeBuilder): + """A Beautiful Soup treebuilder that listens for SAX events.""" + + def feed(self, markup): + raise NotImplementedError() + + def close(self): + pass + + def startElement(self, name, attrs): + attrs = dict((key[1], value) for key, value in list(attrs.items())) + #print "Start %s, %r" % (name, attrs) + self.soup.handle_starttag(name, attrs) + + def endElement(self, name): + #print "End %s" % name + self.soup.handle_endtag(name) + + def startElementNS(self, nsTuple, nodeName, attrs): + # Throw away (ns, nodeName) for now. + self.startElement(nodeName, attrs) + + def endElementNS(self, nsTuple, nodeName): + # Throw away (ns, nodeName) for now. + self.endElement(nodeName) + #handler.endElementNS((ns, node.nodeName), node.nodeName) + + def startPrefixMapping(self, prefix, nodeValue): + # Ignore the prefix for now. + pass + + def endPrefixMapping(self, prefix): + # Ignore the prefix for now. + # handler.endPrefixMapping(prefix) + pass + + def characters(self, content): + self.soup.handle_data(content) + + def startDocument(self): + pass + + def endDocument(self): + pass + + +class HTMLTreeBuilder(TreeBuilder): + """This TreeBuilder knows facts about HTML. + + Such as which tags are empty-element tags. + """ + + preserve_whitespace_tags = set(['pre', 'textarea']) + empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta', + 'spacer', 'link', 'frame', 'base']) + + # The HTML standard defines these attributes as containing a + # space-separated list of values, not a single value. That is, + # class="foo bar" means that the 'class' attribute has two values, + # 'foo' and 'bar', not the single value 'foo bar'. When we + # encounter one of these attributes, we will parse its value into + # a list of values if possible. Upon output, the list will be + # converted back into a string. + cdata_list_attributes = { + "*" : ['class', 'accesskey', 'dropzone'], + "a" : ['rel', 'rev'], + "link" : ['rel', 'rev'], + "td" : ["headers"], + "th" : ["headers"], + "td" : ["headers"], + "form" : ["accept-charset"], + "object" : ["archive"], + + # These are HTML5 specific, as are *.accesskey and *.dropzone above. + "area" : ["rel"], + "icon" : ["sizes"], + "iframe" : ["sandbox"], + "output" : ["for"], + } + + def set_up_substitutions(self, tag): + # We are only interested in tags + if tag.name != 'meta': + return False + + http_equiv = tag.get('http-equiv') + content = tag.get('content') + charset = tag.get('charset') + + # We are interested in tags that say what encoding the + # document was originally in. This means HTML 5-style + # tags that provide the "charset" attribute. It also means + # HTML 4-style tags that provide the "content" + # attribute and have "http-equiv" set to "content-type". + # + # In both cases we will replace the value of the appropriate + # attribute with a standin object that can take on any + # encoding. + meta_encoding = None + if charset is not None: + # HTML 5 style: + # + meta_encoding = charset + tag['charset'] = CharsetMetaAttributeValue(charset) + + elif (content is not None and http_equiv is not None + and http_equiv.lower() == 'content-type'): + # HTML 4 style: + # + tag['content'] = ContentMetaAttributeValue(content) + + return (meta_encoding is not None) + +def register_treebuilders_from(module): + """Copy TreeBuilders from the given module into this module.""" + # I'm fairly sure this is not the best way to do this. + this_module = sys.modules['bs4.builder'] + for name in module.__all__: + obj = getattr(module, name) + + if issubclass(obj, TreeBuilder): + setattr(this_module, name, obj) + this_module.__all__.append(name) + # Register the builder while we're at it. + this_module.builder_registry.register(obj) + +class ParserRejectedMarkup(Exception): + pass + +# Builders are registered in reverse order of priority, so that custom +# builder registrations will take precedence. In general, we want lxml +# to take precedence over html5lib, because it's faster. And we only +# want to use HTMLParser as a last result. +from . import _htmlparser +register_treebuilders_from(_htmlparser) +try: + from . import _html5lib + register_treebuilders_from(_html5lib) +except ImportError: + # They don't have html5lib installed. + pass +try: + from . import _lxml + register_treebuilders_from(_lxml) +except ImportError: + # They don't have lxml installed. + pass diff --git a/bs4/builder/_html5lib.py b/bs4/builder/_html5lib.py new file mode 100644 index 0000000..7de36ae --- /dev/null +++ b/bs4/builder/_html5lib.py @@ -0,0 +1,285 @@ +__all__ = [ + 'HTML5TreeBuilder', + ] + +import warnings +from bs4.builder import ( + PERMISSIVE, + HTML, + HTML_5, + HTMLTreeBuilder, + ) +from bs4.element import NamespacedAttribute +import html5lib +from html5lib.constants import namespaces +from bs4.element import ( + Comment, + Doctype, + NavigableString, + Tag, + ) + +class HTML5TreeBuilder(HTMLTreeBuilder): + """Use html5lib to build a tree.""" + + features = ['html5lib', PERMISSIVE, HTML_5, HTML] + + def prepare_markup(self, markup, user_specified_encoding): + # Store the user-specified encoding for use later on. + self.user_specified_encoding = user_specified_encoding + yield (markup, None, None, False) + + # These methods are defined by Beautiful Soup. + def feed(self, markup): + if self.soup.parse_only is not None: + warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.") + parser = html5lib.HTMLParser(tree=self.create_treebuilder) + doc = parser.parse(markup, encoding=self.user_specified_encoding) + + # Set the character encoding detected by the tokenizer. + if isinstance(markup, unicode): + # We need to special-case this because html5lib sets + # charEncoding to UTF-8 if it gets Unicode input. + doc.original_encoding = None + else: + doc.original_encoding = parser.tokenizer.stream.charEncoding[0] + + def create_treebuilder(self, namespaceHTMLElements): + self.underlying_builder = TreeBuilderForHtml5lib( + self.soup, namespaceHTMLElements) + return self.underlying_builder + + def test_fragment_to_document(self, fragment): + """See `TreeBuilder`.""" + return u'%s' % fragment + + +class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder): + + def __init__(self, soup, namespaceHTMLElements): + self.soup = soup + super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements) + + def documentClass(self): + self.soup.reset() + return Element(self.soup, self.soup, None) + + def insertDoctype(self, token): + name = token["name"] + publicId = token["publicId"] + systemId = token["systemId"] + + doctype = Doctype.for_name_and_ids(name, publicId, systemId) + self.soup.object_was_parsed(doctype) + + def elementClass(self, name, namespace): + tag = self.soup.new_tag(name, namespace) + return Element(tag, self.soup, namespace) + + def commentClass(self, data): + return TextNode(Comment(data), self.soup) + + def fragmentClass(self): + self.soup = BeautifulSoup("") + self.soup.name = "[document_fragment]" + return Element(self.soup, self.soup, None) + + def appendChild(self, node): + # XXX This code is not covered by the BS4 tests. + self.soup.append(node.element) + + def getDocument(self): + return self.soup + + def getFragment(self): + return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element + +class AttrList(object): + def __init__(self, element): + self.element = element + self.attrs = dict(self.element.attrs) + def __iter__(self): + return list(self.attrs.items()).__iter__() + def __setitem__(self, name, value): + "set attr", name, value + self.element[name] = value + def items(self): + return list(self.attrs.items()) + def keys(self): + return list(self.attrs.keys()) + def __len__(self): + return len(self.attrs) + def __getitem__(self, name): + return self.attrs[name] + def __contains__(self, name): + return name in list(self.attrs.keys()) + + +class Element(html5lib.treebuilders._base.Node): + def __init__(self, element, soup, namespace): + html5lib.treebuilders._base.Node.__init__(self, element.name) + self.element = element + self.soup = soup + self.namespace = namespace + + def appendChild(self, node): + string_child = child = None + if isinstance(node, basestring): + # Some other piece of code decided to pass in a string + # instead of creating a TextElement object to contain the + # string. + string_child = child = node + elif isinstance(node, Tag): + # Some other piece of code decided to pass in a Tag + # instead of creating an Element object to contain the + # Tag. + child = node + elif node.element.__class__ == NavigableString: + string_child = child = node.element + else: + child = node.element + + if not isinstance(child, basestring) and child.parent is not None: + node.element.extract() + + if (string_child and self.element.contents + and self.element.contents[-1].__class__ == NavigableString): + # We are appending a string onto another string. + # TODO This has O(n^2) performance, for input like + # "aaa..." + old_element = self.element.contents[-1] + new_element = self.soup.new_string(old_element + string_child) + old_element.replace_with(new_element) + self.soup._most_recent_element = new_element + else: + if isinstance(node, basestring): + # Create a brand new NavigableString from this string. + child = self.soup.new_string(node) + + # Tell Beautiful Soup to act as if it parsed this element + # immediately after the parent's last descendant. (Or + # immediately after the parent, if it has no children.) + if self.element.contents: + most_recent_element = self.element._last_descendant(False) + else: + most_recent_element = self.element + + self.soup.object_was_parsed( + child, parent=self.element, + most_recent_element=most_recent_element) + + def getAttributes(self): + return AttrList(self.element) + + def setAttributes(self, attributes): + if attributes is not None and len(attributes) > 0: + + converted_attributes = [] + for name, value in list(attributes.items()): + if isinstance(name, tuple): + new_name = NamespacedAttribute(*name) + del attributes[name] + attributes[new_name] = value + + self.soup.builder._replace_cdata_list_attribute_values( + self.name, attributes) + for name, value in attributes.items(): + self.element[name] = value + + # The attributes may contain variables that need substitution. + # Call set_up_substitutions manually. + # + # The Tag constructor called this method when the Tag was created, + # but we just set/changed the attributes, so call it again. + self.soup.builder.set_up_substitutions(self.element) + attributes = property(getAttributes, setAttributes) + + def insertText(self, data, insertBefore=None): + if insertBefore: + text = TextNode(self.soup.new_string(data), self.soup) + self.insertBefore(data, insertBefore) + else: + self.appendChild(data) + + def insertBefore(self, node, refNode): + index = self.element.index(refNode.element) + if (node.element.__class__ == NavigableString and self.element.contents + and self.element.contents[index-1].__class__ == NavigableString): + # (See comments in appendChild) + old_node = self.element.contents[index-1] + new_str = self.soup.new_string(old_node + node.element) + old_node.replace_with(new_str) + else: + self.element.insert(index, node.element) + node.parent = self + + def removeChild(self, node): + node.element.extract() + + def reparentChildren(self, new_parent): + """Move all of this tag's children into another tag.""" + element = self.element + new_parent_element = new_parent.element + # Determine what this tag's next_element will be once all the children + # are removed. + final_next_element = element.next_sibling + + new_parents_last_descendant = new_parent_element._last_descendant(False, False) + if len(new_parent_element.contents) > 0: + # The new parent already contains children. We will be + # appending this tag's children to the end. + new_parents_last_child = new_parent_element.contents[-1] + new_parents_last_descendant_next_element = new_parents_last_descendant.next_element + else: + # The new parent contains no children. + new_parents_last_child = None + new_parents_last_descendant_next_element = new_parent_element.next_element + + to_append = element.contents + append_after = new_parent.element.contents + if len(to_append) > 0: + # Set the first child's previous_element and previous_sibling + # to elements within the new parent + first_child = to_append[0] + first_child.previous_element = new_parents_last_descendant + first_child.previous_sibling = new_parents_last_child + + # Fix the last child's next_element and next_sibling + last_child = to_append[-1] + last_child.next_element = new_parents_last_descendant_next_element + last_child.next_sibling = None + + for child in to_append: + child.parent = new_parent_element + new_parent_element.contents.append(child) + + # Now that this element has no children, change its .next_element. + element.contents = [] + element.next_element = final_next_element + + def cloneNode(self): + tag = self.soup.new_tag(self.element.name, self.namespace) + node = Element(tag, self.soup, self.namespace) + for key,value in self.attributes: + node.attributes[key] = value + return node + + def hasContent(self): + return self.element.contents + + def getNameTuple(self): + if self.namespace == None: + return namespaces["html"], self.name + else: + return self.namespace, self.name + + nameTuple = property(getNameTuple) + +class TextNode(Element): + def __init__(self, element, soup): + html5lib.treebuilders._base.Node.__init__(self, None) + self.element = element + self.soup = soup + + def cloneNode(self): + raise NotImplementedError diff --git a/bs4/builder/_htmlparser.py b/bs4/builder/_htmlparser.py new file mode 100644 index 0000000..ca8d8b8 --- /dev/null +++ b/bs4/builder/_htmlparser.py @@ -0,0 +1,258 @@ +"""Use the HTMLParser library to parse HTML files that aren't too bad.""" + +__all__ = [ + 'HTMLParserTreeBuilder', + ] + +from HTMLParser import ( + HTMLParser, + HTMLParseError, + ) +import sys +import warnings + +# Starting in Python 3.2, the HTMLParser constructor takes a 'strict' +# argument, which we'd like to set to False. Unfortunately, +# http://bugs.python.org/issue13273 makes strict=True a better bet +# before Python 3.2.3. +# +# At the end of this file, we monkeypatch HTMLParser so that +# strict=True works well on Python 3.2.2. +major, minor, release = sys.version_info[:3] +CONSTRUCTOR_TAKES_STRICT = ( + major > 3 + or (major == 3 and minor > 2) + or (major == 3 and minor == 2 and release >= 3)) + +from bs4.element import ( + CData, + Comment, + Declaration, + Doctype, + ProcessingInstruction, + ) +from bs4.dammit import EntitySubstitution, UnicodeDammit + +from bs4.builder import ( + HTML, + HTMLTreeBuilder, + STRICT, + ) + + +HTMLPARSER = 'html.parser' + +class BeautifulSoupHTMLParser(HTMLParser): + def handle_starttag(self, name, attrs): + # XXX namespace + attr_dict = {} + for key, value in attrs: + # Change None attribute values to the empty string + # for consistency with the other tree builders. + if value is None: + value = '' + attr_dict[key] = value + attrvalue = '""' + self.soup.handle_starttag(name, None, None, attr_dict) + + def handle_endtag(self, name): + self.soup.handle_endtag(name) + + def handle_data(self, data): + self.soup.handle_data(data) + + def handle_charref(self, name): + # XXX workaround for a bug in HTMLParser. Remove this once + # it's fixed. + if name.startswith('x'): + real_name = int(name.lstrip('x'), 16) + elif name.startswith('X'): + real_name = int(name.lstrip('X'), 16) + else: + real_name = int(name) + + try: + data = unichr(real_name) + except (ValueError, OverflowError), e: + data = u"\N{REPLACEMENT CHARACTER}" + + self.handle_data(data) + + def handle_entityref(self, name): + character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name) + if character is not None: + data = character + else: + data = "&%s;" % name + self.handle_data(data) + + def handle_comment(self, data): + self.soup.endData() + self.soup.handle_data(data) + self.soup.endData(Comment) + + def handle_decl(self, data): + self.soup.endData() + if data.startswith("DOCTYPE "): + data = data[len("DOCTYPE "):] + elif data == 'DOCTYPE': + # i.e. "" + data = '' + self.soup.handle_data(data) + self.soup.endData(Doctype) + + def unknown_decl(self, data): + if data.upper().startswith('CDATA['): + cls = CData + data = data[len('CDATA['):] + else: + cls = Declaration + self.soup.endData() + self.soup.handle_data(data) + self.soup.endData(cls) + + def handle_pi(self, data): + self.soup.endData() + if data.endswith("?") and data.lower().startswith("xml"): + # "An XHTML processing instruction using the trailing '?' + # will cause the '?' to be included in data." - HTMLParser + # docs. + # + # Strip the question mark so we don't end up with two + # question marks. + data = data[:-1] + self.soup.handle_data(data) + self.soup.endData(ProcessingInstruction) + + +class HTMLParserTreeBuilder(HTMLTreeBuilder): + + is_xml = False + features = [HTML, STRICT, HTMLPARSER] + + def __init__(self, *args, **kwargs): + if CONSTRUCTOR_TAKES_STRICT: + kwargs['strict'] = False + self.parser_args = (args, kwargs) + + def prepare_markup(self, markup, user_specified_encoding=None, + document_declared_encoding=None): + """ + :return: A 4-tuple (markup, original encoding, encoding + declared within markup, whether any characters had to be + replaced with REPLACEMENT CHARACTER). + """ + if isinstance(markup, unicode): + yield (markup, None, None, False) + return + + try_encodings = [user_specified_encoding, document_declared_encoding] + dammit = UnicodeDammit(markup, try_encodings, is_html=True) + yield (dammit.markup, dammit.original_encoding, + dammit.declared_html_encoding, + dammit.contains_replacement_characters) + + def feed(self, markup): + args, kwargs = self.parser_args + parser = BeautifulSoupHTMLParser(*args, **kwargs) + parser.soup = self.soup + try: + parser.feed(markup) + except HTMLParseError, e: + warnings.warn(RuntimeWarning( + "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help.")) + raise e + +# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some +# 3.2.3 code. This ensures they don't treat markup like

as a +# string. +# +# XXX This code can be removed once most Python 3 users are on 3.2.3. +if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT: + import re + attrfind_tolerant = re.compile( + r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*' + r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?') + HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant + + locatestarttagend = re.compile(r""" + <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name + (?:\s+ # whitespace before attribute name + (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name + (?:\s*=\s* # value indicator + (?:'[^']*' # LITA-enclosed value + |\"[^\"]*\" # LIT-enclosed value + |[^'\">\s]+ # bare value + ) + )? + ) + )* + \s* # trailing whitespace +""", re.VERBOSE) + BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend + + from html.parser import tagfind, attrfind + + def parse_starttag(self, i): + self.__starttag_text = None + endpos = self.check_for_whole_start_tag(i) + if endpos < 0: + return endpos + rawdata = self.rawdata + self.__starttag_text = rawdata[i:endpos] + + # Now parse the data between i+1 and j into a tag and attrs + attrs = [] + match = tagfind.match(rawdata, i+1) + assert match, 'unexpected call to parse_starttag()' + k = match.end() + self.lasttag = tag = rawdata[i+1:k].lower() + while k < endpos: + if self.strict: + m = attrfind.match(rawdata, k) + else: + m = attrfind_tolerant.match(rawdata, k) + if not m: + break + attrname, rest, attrvalue = m.group(1, 2, 3) + if not rest: + attrvalue = None + elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ + attrvalue[:1] == '"' == attrvalue[-1:]: + attrvalue = attrvalue[1:-1] + if attrvalue: + attrvalue = self.unescape(attrvalue) + attrs.append((attrname.lower(), attrvalue)) + k = m.end() + + end = rawdata[k:endpos].strip() + if end not in (">", "/>"): + lineno, offset = self.getpos() + if "\n" in self.__starttag_text: + lineno = lineno + self.__starttag_text.count("\n") + offset = len(self.__starttag_text) \ + - self.__starttag_text.rfind("\n") + else: + offset = offset + len(self.__starttag_text) + if self.strict: + self.error("junk characters in start tag: %r" + % (rawdata[k:endpos][:20],)) + self.handle_data(rawdata[i:endpos]) + return endpos + if end.endswith('/>'): + # XHTML-style empty tag: + self.handle_startendtag(tag, attrs) + else: + self.handle_starttag(tag, attrs) + if tag in self.CDATA_CONTENT_ELEMENTS: + self.set_cdata_mode(tag) + return endpos + + def set_cdata_mode(self, elem): + self.cdata_elem = elem.lower() + self.interesting = re.compile(r'' % self.cdata_elem, re.I) + + BeautifulSoupHTMLParser.parse_starttag = parse_starttag + BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode + + CONSTRUCTOR_TAKES_STRICT = True diff --git a/bs4/builder/_lxml.py b/bs4/builder/_lxml.py new file mode 100644 index 0000000..fa5d498 --- /dev/null +++ b/bs4/builder/_lxml.py @@ -0,0 +1,233 @@ +__all__ = [ + 'LXMLTreeBuilderForXML', + 'LXMLTreeBuilder', + ] + +from io import BytesIO +from StringIO import StringIO +import collections +from lxml import etree +from bs4.element import Comment, Doctype, NamespacedAttribute +from bs4.builder import ( + FAST, + HTML, + HTMLTreeBuilder, + PERMISSIVE, + ParserRejectedMarkup, + TreeBuilder, + XML) +from bs4.dammit import EncodingDetector + +LXML = 'lxml' + +class LXMLTreeBuilderForXML(TreeBuilder): + DEFAULT_PARSER_CLASS = etree.XMLParser + + is_xml = True + + # Well, it's permissive by XML parser standards. + features = [LXML, XML, FAST, PERMISSIVE] + + CHUNK_SIZE = 512 + + # This namespace mapping is specified in the XML Namespace + # standard. + DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"} + + def default_parser(self, encoding): + # This can either return a parser object or a class, which + # will be instantiated with default arguments. + if self._default_parser is not None: + return self._default_parser + return etree.XMLParser( + target=self, strip_cdata=False, recover=True, encoding=encoding) + + def parser_for(self, encoding): + # Use the default parser. + parser = self.default_parser(encoding) + + if isinstance(parser, collections.Callable): + # Instantiate the parser with default arguments + parser = parser(target=self, strip_cdata=False, encoding=encoding) + return parser + + def __init__(self, parser=None, empty_element_tags=None): + # TODO: Issue a warning if parser is present but not a + # callable, since that means there's no way to create new + # parsers for different encodings. + self._default_parser = parser + if empty_element_tags is not None: + self.empty_element_tags = set(empty_element_tags) + self.soup = None + self.nsmaps = [self.DEFAULT_NSMAPS] + + def _getNsTag(self, tag): + # Split the namespace URL out of a fully-qualified lxml tag + # name. Copied from lxml's src/lxml/sax.py. + if tag[0] == '{': + return tuple(tag[1:].split('}', 1)) + else: + return (None, tag) + + def prepare_markup(self, markup, user_specified_encoding=None, + document_declared_encoding=None): + """ + :yield: A series of 4-tuples. + (markup, encoding, declared encoding, + has undergone character replacement) + + Each 4-tuple represents a strategy for parsing the document. + """ + if isinstance(markup, unicode): + # We were given Unicode. Maybe lxml can parse Unicode on + # this system? + yield markup, None, document_declared_encoding, False + + if isinstance(markup, unicode): + # No, apparently not. Convert the Unicode to UTF-8 and + # tell lxml to parse it as UTF-8. + yield (markup.encode("utf8"), "utf8", + document_declared_encoding, False) + + # Instead of using UnicodeDammit to convert the bytestring to + # Unicode using different encodings, use EncodingDetector to + # iterate over the encodings, and tell lxml to try to parse + # the document as each one in turn. + is_html = not self.is_xml + try_encodings = [user_specified_encoding, document_declared_encoding] + detector = EncodingDetector(markup, try_encodings, is_html) + for encoding in detector.encodings: + yield (detector.markup, encoding, document_declared_encoding, False) + + def feed(self, markup): + if isinstance(markup, bytes): + markup = BytesIO(markup) + elif isinstance(markup, unicode): + markup = StringIO(markup) + + # Call feed() at least once, even if the markup is empty, + # or the parser won't be initialized. + data = markup.read(self.CHUNK_SIZE) + try: + self.parser = self.parser_for(self.soup.original_encoding) + self.parser.feed(data) + while len(data) != 0: + # Now call feed() on the rest of the data, chunk by chunk. + data = markup.read(self.CHUNK_SIZE) + if len(data) != 0: + self.parser.feed(data) + self.parser.close() + except (UnicodeDecodeError, LookupError, etree.ParserError), e: + raise ParserRejectedMarkup(str(e)) + + def close(self): + self.nsmaps = [self.DEFAULT_NSMAPS] + + def start(self, name, attrs, nsmap={}): + # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy. + attrs = dict(attrs) + nsprefix = None + # Invert each namespace map as it comes in. + if len(self.nsmaps) > 1: + # There are no new namespaces for this tag, but + # non-default namespaces are in play, so we need a + # separate tag stack to know when they end. + self.nsmaps.append(None) + elif len(nsmap) > 0: + # A new namespace mapping has come into play. + inverted_nsmap = dict((value, key) for key, value in nsmap.items()) + self.nsmaps.append(inverted_nsmap) + # Also treat the namespace mapping as a set of attributes on the + # tag, so we can recreate it later. + attrs = attrs.copy() + for prefix, namespace in nsmap.items(): + attribute = NamespacedAttribute( + "xmlns", prefix, "http://www.w3.org/2000/xmlns/") + attrs[attribute] = namespace + + # Namespaces are in play. Find any attributes that came in + # from lxml with namespaces attached to their names, and + # turn then into NamespacedAttribute objects. + new_attrs = {} + for attr, value in attrs.items(): + namespace, attr = self._getNsTag(attr) + if namespace is None: + new_attrs[attr] = value + else: + nsprefix = self._prefix_for_namespace(namespace) + attr = NamespacedAttribute(nsprefix, attr, namespace) + new_attrs[attr] = value + attrs = new_attrs + + namespace, name = self._getNsTag(name) + nsprefix = self._prefix_for_namespace(namespace) + self.soup.handle_starttag(name, namespace, nsprefix, attrs) + + def _prefix_for_namespace(self, namespace): + """Find the currently active prefix for the given namespace.""" + if namespace is None: + return None + for inverted_nsmap in reversed(self.nsmaps): + if inverted_nsmap is not None and namespace in inverted_nsmap: + return inverted_nsmap[namespace] + return None + + def end(self, name): + self.soup.endData() + completed_tag = self.soup.tagStack[-1] + namespace, name = self._getNsTag(name) + nsprefix = None + if namespace is not None: + for inverted_nsmap in reversed(self.nsmaps): + if inverted_nsmap is not None and namespace in inverted_nsmap: + nsprefix = inverted_nsmap[namespace] + break + self.soup.handle_endtag(name, nsprefix) + if len(self.nsmaps) > 1: + # This tag, or one of its parents, introduced a namespace + # mapping, so pop it off the stack. + self.nsmaps.pop() + + def pi(self, target, data): + pass + + def data(self, content): + self.soup.handle_data(content) + + def doctype(self, name, pubid, system): + self.soup.endData() + doctype = Doctype.for_name_and_ids(name, pubid, system) + self.soup.object_was_parsed(doctype) + + def comment(self, content): + "Handle comments as Comment objects." + self.soup.endData() + self.soup.handle_data(content) + self.soup.endData(Comment) + + def test_fragment_to_document(self, fragment): + """See `TreeBuilder`.""" + return u'\n%s' % fragment + + +class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML): + + features = [LXML, HTML, FAST, PERMISSIVE] + is_xml = False + + def default_parser(self, encoding): + return etree.HTMLParser + + def feed(self, markup): + encoding = self.soup.original_encoding + try: + self.parser = self.parser_for(encoding) + self.parser.feed(markup) + self.parser.close() + except (UnicodeDecodeError, LookupError, etree.ParserError), e: + raise ParserRejectedMarkup(str(e)) + + + def test_fragment_to_document(self, fragment): + """See `TreeBuilder`.""" + return u'%s' % fragment diff --git a/bs4/dammit.py b/bs4/dammit.py new file mode 100644 index 0000000..59640b7 --- /dev/null +++ b/bs4/dammit.py @@ -0,0 +1,829 @@ +# -*- coding: utf-8 -*- +"""Beautiful Soup bonus library: Unicode, Dammit + +This library converts a bytestream to Unicode through any means +necessary. It is heavily based on code from Mark Pilgrim's Universal +Feed Parser. It works best on XML and XML, but it does not rewrite the +XML or HTML to reflect a new encoding; that's the tree builder's job. +""" + +import codecs +from htmlentitydefs import codepoint2name +import re +import logging +import string + +# Import a library to autodetect character encodings. +chardet_type = None +try: + # First try the fast C implementation. + # PyPI package: cchardet + import cchardet + def chardet_dammit(s): + return cchardet.detect(s)['encoding'] +except ImportError: + try: + # Fall back to the pure Python implementation + # Debian package: python-chardet + # PyPI package: chardet + import chardet + def chardet_dammit(s): + return chardet.detect(s)['encoding'] + #import chardet.constants + #chardet.constants._debug = 1 + except ImportError: + # No chardet available. + def chardet_dammit(s): + return None + +# Available from http://cjkpython.i18n.org/. +try: + import iconv_codec +except ImportError: + pass + +xml_encoding_re = re.compile( + '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I) +html_meta_re = re.compile( + '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I) + +class EntitySubstitution(object): + + """Substitute XML or HTML entities for the corresponding characters.""" + + def _populate_class_variables(): + lookup = {} + reverse_lookup = {} + characters_for_re = [] + for codepoint, name in list(codepoint2name.items()): + character = unichr(codepoint) + if codepoint != 34: + # There's no point in turning the quotation mark into + # ", unless it happens within an attribute value, which + # is handled elsewhere. + characters_for_re.append(character) + lookup[character] = name + # But we do want to turn " into the quotation mark. + reverse_lookup[name] = character + re_definition = "[%s]" % "".join(characters_for_re) + return lookup, reverse_lookup, re.compile(re_definition) + (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER, + CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables() + + CHARACTER_TO_XML_ENTITY = { + "'": "apos", + '"': "quot", + "&": "amp", + "<": "lt", + ">": "gt", + } + + BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" + ")") + + AMPERSAND_OR_BRACKET = re.compile("([<>&])") + + @classmethod + def _substitute_html_entity(cls, matchobj): + entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0)) + return "&%s;" % entity + + @classmethod + def _substitute_xml_entity(cls, matchobj): + """Used with a regular expression to substitute the + appropriate XML entity for an XML special character.""" + entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)] + return "&%s;" % entity + + @classmethod + def quoted_attribute_value(self, value): + """Make a value into a quoted XML attribute, possibly escaping it. + + Most strings will be quoted using double quotes. + + Bob's Bar -> "Bob's Bar" + + If a string contains double quotes, it will be quoted using + single quotes. + + Welcome to "my bar" -> 'Welcome to "my bar"' + + If a string contains both single and double quotes, the + double quotes will be escaped, and the string will be quoted + using double quotes. + + Welcome to "Bob's Bar" -> "Welcome to "Bob's bar" + """ + quote_with = '"' + if '"' in value: + if "'" in value: + # The string contains both single and double + # quotes. Turn the double quotes into + # entities. We quote the double quotes rather than + # the single quotes because the entity name is + # """ whether this is HTML or XML. If we + # quoted the single quotes, we'd have to decide + # between ' and &squot;. + replace_with = """ + value = value.replace('"', replace_with) + else: + # There are double quotes but no single quotes. + # We can use single quotes to quote the attribute. + quote_with = "'" + return quote_with + value + quote_with + + @classmethod + def substitute_xml(cls, value, make_quoted_attribute=False): + """Substitute XML entities for special XML characters. + + :param value: A string to be substituted. The less-than sign + will become <, the greater-than sign will become >, + and any ampersands will become &. If you want ampersands + that appear to be part of an entity definition to be left + alone, use substitute_xml_containing_entities() instead. + + :param make_quoted_attribute: If True, then the string will be + quoted, as befits an attribute value. + """ + # Escape angle brackets and ampersands. + value = cls.AMPERSAND_OR_BRACKET.sub( + cls._substitute_xml_entity, value) + + if make_quoted_attribute: + value = cls.quoted_attribute_value(value) + return value + + @classmethod + def substitute_xml_containing_entities( + cls, value, make_quoted_attribute=False): + """Substitute XML entities for special XML characters. + + :param value: A string to be substituted. The less-than sign will + become <, the greater-than sign will become >, and any + ampersands that are not part of an entity defition will + become &. + + :param make_quoted_attribute: If True, then the string will be + quoted, as befits an attribute value. + """ + # Escape angle brackets, and ampersands that aren't part of + # entities. + value = cls.BARE_AMPERSAND_OR_BRACKET.sub( + cls._substitute_xml_entity, value) + + if make_quoted_attribute: + value = cls.quoted_attribute_value(value) + return value + + @classmethod + def substitute_html(cls, s): + """Replace certain Unicode characters with named HTML entities. + + This differs from data.encode(encoding, 'xmlcharrefreplace') + in that the goal is to make the result more readable (to those + with ASCII displays) rather than to recover from + errors. There's absolutely nothing wrong with a UTF-8 string + containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that + character with "é" will make it more readable to some + people. + """ + return cls.CHARACTER_TO_HTML_ENTITY_RE.sub( + cls._substitute_html_entity, s) + + +class EncodingDetector: + """Suggests a number of possible encodings for a bytestring. + + Order of precedence: + + 1. Encodings you specifically tell EncodingDetector to try first + (the override_encodings argument to the constructor). + + 2. An encoding declared within the bytestring itself, either in an + XML declaration (if the bytestring is to be interpreted as an XML + document), or in a tag (if the bytestring is to be + interpreted as an HTML document.) + + 3. An encoding detected through textual analysis by chardet, + cchardet, or a similar external library. + + 4. UTF-8. + + 5. Windows-1252. + """ + def __init__(self, markup, override_encodings=None, is_html=False): + self.override_encodings = override_encodings or [] + self.chardet_encoding = None + self.is_html = is_html + self.declared_encoding = None + + # First order of business: strip a byte-order mark. + self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup) + + def _usable(self, encoding, tried): + if encoding is not None: + encoding = encoding.lower() + if encoding not in tried: + tried.add(encoding) + return True + return False + + @property + def encodings(self): + """Yield a number of encodings that might work for this markup.""" + tried = set() + for e in self.override_encodings: + if self._usable(e, tried): + yield e + + # Did the document originally start with a byte-order mark + # that indicated its encoding? + if self._usable(self.sniffed_encoding, tried): + yield self.sniffed_encoding + + # Look within the document for an XML or HTML encoding + # declaration. + if self.declared_encoding is None: + self.declared_encoding = self.find_declared_encoding( + self.markup, self.is_html) + if self._usable(self.declared_encoding, tried): + yield self.declared_encoding + + # Use third-party character set detection to guess at the + # encoding. + if self.chardet_encoding is None: + self.chardet_encoding = chardet_dammit(self.markup) + if self._usable(self.chardet_encoding, tried): + yield self.chardet_encoding + + # As a last-ditch effort, try utf-8 and windows-1252. + for e in ('utf-8', 'windows-1252'): + if self._usable(e, tried): + yield e + + @classmethod + def strip_byte_order_mark(cls, data): + """If a byte-order mark is present, strip it and return the encoding it implies.""" + encoding = None + if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \ + and (data[2:4] != '\x00\x00'): + encoding = 'utf-16be' + data = data[2:] + elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \ + and (data[2:4] != '\x00\x00'): + encoding = 'utf-16le' + data = data[2:] + elif data[:3] == b'\xef\xbb\xbf': + encoding = 'utf-8' + data = data[3:] + elif data[:4] == b'\x00\x00\xfe\xff': + encoding = 'utf-32be' + data = data[4:] + elif data[:4] == b'\xff\xfe\x00\x00': + encoding = 'utf-32le' + data = data[4:] + return data, encoding + + @classmethod + def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False): + """Given a document, tries to find its declared encoding. + + An XML encoding is declared at the beginning of the document. + + An HTML encoding is declared in a tag, hopefully near the + beginning of the document. + """ + if search_entire_document: + xml_endpos = html_endpos = len(markup) + else: + xml_endpos = 1024 + html_endpos = max(2048, int(len(markup) * 0.05)) + + declared_encoding = None + declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos) + if not declared_encoding_match and is_html: + declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos) + if declared_encoding_match is not None: + declared_encoding = declared_encoding_match.groups()[0].decode( + 'ascii') + if declared_encoding: + return declared_encoding.lower() + return None + +class UnicodeDammit: + """A class for detecting the encoding of a *ML document and + converting it to a Unicode string. If the source encoding is + windows-1252, can replace MS smart quotes with their HTML or XML + equivalents.""" + + # This dictionary maps commonly seen values for "charset" in HTML + # meta tags to the corresponding Python codec names. It only covers + # values that aren't in Python's aliases and can't be determined + # by the heuristics in find_codec. + CHARSET_ALIASES = {"macintosh": "mac-roman", + "x-sjis": "shift-jis"} + + ENCODINGS_WITH_SMART_QUOTES = [ + "windows-1252", + "iso-8859-1", + "iso-8859-2", + ] + + def __init__(self, markup, override_encodings=[], + smart_quotes_to=None, is_html=False): + self.smart_quotes_to = smart_quotes_to + self.tried_encodings = [] + self.contains_replacement_characters = False + self.is_html = is_html + + self.detector = EncodingDetector(markup, override_encodings, is_html) + + # Short-circuit if the data is in Unicode to begin with. + if isinstance(markup, unicode) or markup == '': + self.markup = markup + self.unicode_markup = unicode(markup) + self.original_encoding = None + return + + # The encoding detector may have stripped a byte-order mark. + # Use the stripped markup from this point on. + self.markup = self.detector.markup + + u = None + for encoding in self.detector.encodings: + markup = self.detector.markup + u = self._convert_from(encoding) + if u is not None: + break + + if not u: + # None of the encodings worked. As an absolute last resort, + # try them again with character replacement. + + for encoding in self.detector.encodings: + if encoding != "ascii": + u = self._convert_from(encoding, "replace") + if u is not None: + logging.warning( + "Some characters could not be decoded, and were " + "replaced with REPLACEMENT CHARACTER.") + self.contains_replacement_characters = True + break + + # If none of that worked, we could at this point force it to + # ASCII, but that would destroy so much data that I think + # giving up is better. + self.unicode_markup = u + if not u: + self.original_encoding = None + + def _sub_ms_char(self, match): + """Changes a MS smart quote character to an XML or HTML + entity, or an ASCII character.""" + orig = match.group(1) + if self.smart_quotes_to == 'ascii': + sub = self.MS_CHARS_TO_ASCII.get(orig).encode() + else: + sub = self.MS_CHARS.get(orig) + if type(sub) == tuple: + if self.smart_quotes_to == 'xml': + sub = '&#x'.encode() + sub[1].encode() + ';'.encode() + else: + sub = '&'.encode() + sub[0].encode() + ';'.encode() + else: + sub = sub.encode() + return sub + + def _convert_from(self, proposed, errors="strict"): + proposed = self.find_codec(proposed) + if not proposed or (proposed, errors) in self.tried_encodings: + return None + self.tried_encodings.append((proposed, errors)) + markup = self.markup + # Convert smart quotes to HTML if coming from an encoding + # that might have them. + if (self.smart_quotes_to is not None + and proposed in self.ENCODINGS_WITH_SMART_QUOTES): + smart_quotes_re = b"([\x80-\x9f])" + smart_quotes_compiled = re.compile(smart_quotes_re) + markup = smart_quotes_compiled.sub(self._sub_ms_char, markup) + + try: + #print "Trying to convert document to %s (errors=%s)" % ( + # proposed, errors) + u = self._to_unicode(markup, proposed, errors) + self.markup = u + self.original_encoding = proposed + except Exception as e: + #print "That didn't work!" + #print e + return None + #print "Correct encoding: %s" % proposed + return self.markup + + def _to_unicode(self, data, encoding, errors="strict"): + '''Given a string and its encoding, decodes the string into Unicode. + %encoding is a string recognized by encodings.aliases''' + return unicode(data, encoding, errors) + + @property + def declared_html_encoding(self): + if not self.is_html: + return None + return self.detector.declared_encoding + + def find_codec(self, charset): + value = (self._codec(self.CHARSET_ALIASES.get(charset, charset)) + or (charset and self._codec(charset.replace("-", ""))) + or (charset and self._codec(charset.replace("-", "_"))) + or (charset and charset.lower()) + or charset + ) + if value: + return value.lower() + return None + + def _codec(self, charset): + if not charset: + return charset + codec = None + try: + codecs.lookup(charset) + codec = charset + except (LookupError, ValueError): + pass + return codec + + + # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities. + MS_CHARS = {b'\x80': ('euro', '20AC'), + b'\x81': ' ', + b'\x82': ('sbquo', '201A'), + b'\x83': ('fnof', '192'), + b'\x84': ('bdquo', '201E'), + b'\x85': ('hellip', '2026'), + b'\x86': ('dagger', '2020'), + b'\x87': ('Dagger', '2021'), + b'\x88': ('circ', '2C6'), + b'\x89': ('permil', '2030'), + b'\x8A': ('Scaron', '160'), + b'\x8B': ('lsaquo', '2039'), + b'\x8C': ('OElig', '152'), + b'\x8D': '?', + b'\x8E': ('#x17D', '17D'), + b'\x8F': '?', + b'\x90': '?', + b'\x91': ('lsquo', '2018'), + b'\x92': ('rsquo', '2019'), + b'\x93': ('ldquo', '201C'), + b'\x94': ('rdquo', '201D'), + b'\x95': ('bull', '2022'), + b'\x96': ('ndash', '2013'), + b'\x97': ('mdash', '2014'), + b'\x98': ('tilde', '2DC'), + b'\x99': ('trade', '2122'), + b'\x9a': ('scaron', '161'), + b'\x9b': ('rsaquo', '203A'), + b'\x9c': ('oelig', '153'), + b'\x9d': '?', + b'\x9e': ('#x17E', '17E'), + b'\x9f': ('Yuml', ''),} + + # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains + # horrors like stripping diacritical marks to turn á into a, but also + # contains non-horrors like turning “ into ". + MS_CHARS_TO_ASCII = { + b'\x80' : 'EUR', + b'\x81' : ' ', + b'\x82' : ',', + b'\x83' : 'f', + b'\x84' : ',,', + b'\x85' : '...', + b'\x86' : '+', + b'\x87' : '++', + b'\x88' : '^', + b'\x89' : '%', + b'\x8a' : 'S', + b'\x8b' : '<', + b'\x8c' : 'OE', + b'\x8d' : '?', + b'\x8e' : 'Z', + b'\x8f' : '?', + b'\x90' : '?', + b'\x91' : "'", + b'\x92' : "'", + b'\x93' : '"', + b'\x94' : '"', + b'\x95' : '*', + b'\x96' : '-', + b'\x97' : '--', + b'\x98' : '~', + b'\x99' : '(TM)', + b'\x9a' : 's', + b'\x9b' : '>', + b'\x9c' : 'oe', + b'\x9d' : '?', + b'\x9e' : 'z', + b'\x9f' : 'Y', + b'\xa0' : ' ', + b'\xa1' : '!', + b'\xa2' : 'c', + b'\xa3' : 'GBP', + b'\xa4' : '$', #This approximation is especially parochial--this is the + #generic currency symbol. + b'\xa5' : 'YEN', + b'\xa6' : '|', + b'\xa7' : 'S', + b'\xa8' : '..', + b'\xa9' : '', + b'\xaa' : '(th)', + b'\xab' : '<<', + b'\xac' : '!', + b'\xad' : ' ', + b'\xae' : '(R)', + b'\xaf' : '-', + b'\xb0' : 'o', + b'\xb1' : '+-', + b'\xb2' : '2', + b'\xb3' : '3', + b'\xb4' : ("'", 'acute'), + b'\xb5' : 'u', + b'\xb6' : 'P', + b'\xb7' : '*', + b'\xb8' : ',', + b'\xb9' : '1', + b'\xba' : '(th)', + b'\xbb' : '>>', + b'\xbc' : '1/4', + b'\xbd' : '1/2', + b'\xbe' : '3/4', + b'\xbf' : '?', + b'\xc0' : 'A', + b'\xc1' : 'A', + b'\xc2' : 'A', + b'\xc3' : 'A', + b'\xc4' : 'A', + b'\xc5' : 'A', + b'\xc6' : 'AE', + b'\xc7' : 'C', + b'\xc8' : 'E', + b'\xc9' : 'E', + b'\xca' : 'E', + b'\xcb' : 'E', + b'\xcc' : 'I', + b'\xcd' : 'I', + b'\xce' : 'I', + b'\xcf' : 'I', + b'\xd0' : 'D', + b'\xd1' : 'N', + b'\xd2' : 'O', + b'\xd3' : 'O', + b'\xd4' : 'O', + b'\xd5' : 'O', + b'\xd6' : 'O', + b'\xd7' : '*', + b'\xd8' : 'O', + b'\xd9' : 'U', + b'\xda' : 'U', + b'\xdb' : 'U', + b'\xdc' : 'U', + b'\xdd' : 'Y', + b'\xde' : 'b', + b'\xdf' : 'B', + b'\xe0' : 'a', + b'\xe1' : 'a', + b'\xe2' : 'a', + b'\xe3' : 'a', + b'\xe4' : 'a', + b'\xe5' : 'a', + b'\xe6' : 'ae', + b'\xe7' : 'c', + b'\xe8' : 'e', + b'\xe9' : 'e', + b'\xea' : 'e', + b'\xeb' : 'e', + b'\xec' : 'i', + b'\xed' : 'i', + b'\xee' : 'i', + b'\xef' : 'i', + b'\xf0' : 'o', + b'\xf1' : 'n', + b'\xf2' : 'o', + b'\xf3' : 'o', + b'\xf4' : 'o', + b'\xf5' : 'o', + b'\xf6' : 'o', + b'\xf7' : '/', + b'\xf8' : 'o', + b'\xf9' : 'u', + b'\xfa' : 'u', + b'\xfb' : 'u', + b'\xfc' : 'u', + b'\xfd' : 'y', + b'\xfe' : 'b', + b'\xff' : 'y', + } + + # A map used when removing rogue Windows-1252/ISO-8859-1 + # characters in otherwise UTF-8 documents. + # + # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in + # Windows-1252. + WINDOWS_1252_TO_UTF8 = { + 0x80 : b'\xe2\x82\xac', # € + 0x82 : b'\xe2\x80\x9a', # ‚ + 0x83 : b'\xc6\x92', # ƒ + 0x84 : b'\xe2\x80\x9e', # „ + 0x85 : b'\xe2\x80\xa6', # … + 0x86 : b'\xe2\x80\xa0', # † + 0x87 : b'\xe2\x80\xa1', # ‡ + 0x88 : b'\xcb\x86', # ˆ + 0x89 : b'\xe2\x80\xb0', # ‰ + 0x8a : b'\xc5\xa0', # Š + 0x8b : b'\xe2\x80\xb9', # ‹ + 0x8c : b'\xc5\x92', # Œ + 0x8e : b'\xc5\xbd', # Ž + 0x91 : b'\xe2\x80\x98', # ‘ + 0x92 : b'\xe2\x80\x99', # ’ + 0x93 : b'\xe2\x80\x9c', # “ + 0x94 : b'\xe2\x80\x9d', # ” + 0x95 : b'\xe2\x80\xa2', # • + 0x96 : b'\xe2\x80\x93', # – + 0x97 : b'\xe2\x80\x94', # — + 0x98 : b'\xcb\x9c', # ˜ + 0x99 : b'\xe2\x84\xa2', # ™ + 0x9a : b'\xc5\xa1', # š + 0x9b : b'\xe2\x80\xba', # › + 0x9c : b'\xc5\x93', # œ + 0x9e : b'\xc5\xbe', # ž + 0x9f : b'\xc5\xb8', # Ÿ + 0xa0 : b'\xc2\xa0', #   + 0xa1 : b'\xc2\xa1', # ¡ + 0xa2 : b'\xc2\xa2', # ¢ + 0xa3 : b'\xc2\xa3', # £ + 0xa4 : b'\xc2\xa4', # ¤ + 0xa5 : b'\xc2\xa5', # ¥ + 0xa6 : b'\xc2\xa6', # ¦ + 0xa7 : b'\xc2\xa7', # § + 0xa8 : b'\xc2\xa8', # ¨ + 0xa9 : b'\xc2\xa9', # © + 0xaa : b'\xc2\xaa', # ª + 0xab : b'\xc2\xab', # « + 0xac : b'\xc2\xac', # ¬ + 0xad : b'\xc2\xad', # ­ + 0xae : b'\xc2\xae', # ® + 0xaf : b'\xc2\xaf', # ¯ + 0xb0 : b'\xc2\xb0', # ° + 0xb1 : b'\xc2\xb1', # ± + 0xb2 : b'\xc2\xb2', # ² + 0xb3 : b'\xc2\xb3', # ³ + 0xb4 : b'\xc2\xb4', # ´ + 0xb5 : b'\xc2\xb5', # µ + 0xb6 : b'\xc2\xb6', # ¶ + 0xb7 : b'\xc2\xb7', # · + 0xb8 : b'\xc2\xb8', # ¸ + 0xb9 : b'\xc2\xb9', # ¹ + 0xba : b'\xc2\xba', # º + 0xbb : b'\xc2\xbb', # » + 0xbc : b'\xc2\xbc', # ¼ + 0xbd : b'\xc2\xbd', # ½ + 0xbe : b'\xc2\xbe', # ¾ + 0xbf : b'\xc2\xbf', # ¿ + 0xc0 : b'\xc3\x80', # À + 0xc1 : b'\xc3\x81', # Á + 0xc2 : b'\xc3\x82', #  + 0xc3 : b'\xc3\x83', # à + 0xc4 : b'\xc3\x84', # Ä + 0xc5 : b'\xc3\x85', # Å + 0xc6 : b'\xc3\x86', # Æ + 0xc7 : b'\xc3\x87', # Ç + 0xc8 : b'\xc3\x88', # È + 0xc9 : b'\xc3\x89', # É + 0xca : b'\xc3\x8a', # Ê + 0xcb : b'\xc3\x8b', # Ë + 0xcc : b'\xc3\x8c', # Ì + 0xcd : b'\xc3\x8d', # Í + 0xce : b'\xc3\x8e', # Î + 0xcf : b'\xc3\x8f', # Ï + 0xd0 : b'\xc3\x90', # Ð + 0xd1 : b'\xc3\x91', # Ñ + 0xd2 : b'\xc3\x92', # Ò + 0xd3 : b'\xc3\x93', # Ó + 0xd4 : b'\xc3\x94', # Ô + 0xd5 : b'\xc3\x95', # Õ + 0xd6 : b'\xc3\x96', # Ö + 0xd7 : b'\xc3\x97', # × + 0xd8 : b'\xc3\x98', # Ø + 0xd9 : b'\xc3\x99', # Ù + 0xda : b'\xc3\x9a', # Ú + 0xdb : b'\xc3\x9b', # Û + 0xdc : b'\xc3\x9c', # Ü + 0xdd : b'\xc3\x9d', # Ý + 0xde : b'\xc3\x9e', # Þ + 0xdf : b'\xc3\x9f', # ß + 0xe0 : b'\xc3\xa0', # à + 0xe1 : b'\xa1', # á + 0xe2 : b'\xc3\xa2', # â + 0xe3 : b'\xc3\xa3', # ã + 0xe4 : b'\xc3\xa4', # ä + 0xe5 : b'\xc3\xa5', # å + 0xe6 : b'\xc3\xa6', # æ + 0xe7 : b'\xc3\xa7', # ç + 0xe8 : b'\xc3\xa8', # è + 0xe9 : b'\xc3\xa9', # é + 0xea : b'\xc3\xaa', # ê + 0xeb : b'\xc3\xab', # ë + 0xec : b'\xc3\xac', # ì + 0xed : b'\xc3\xad', # í + 0xee : b'\xc3\xae', # î + 0xef : b'\xc3\xaf', # ï + 0xf0 : b'\xc3\xb0', # ð + 0xf1 : b'\xc3\xb1', # ñ + 0xf2 : b'\xc3\xb2', # ò + 0xf3 : b'\xc3\xb3', # ó + 0xf4 : b'\xc3\xb4', # ô + 0xf5 : b'\xc3\xb5', # õ + 0xf6 : b'\xc3\xb6', # ö + 0xf7 : b'\xc3\xb7', # ÷ + 0xf8 : b'\xc3\xb8', # ø + 0xf9 : b'\xc3\xb9', # ù + 0xfa : b'\xc3\xba', # ú + 0xfb : b'\xc3\xbb', # û + 0xfc : b'\xc3\xbc', # ü + 0xfd : b'\xc3\xbd', # ý + 0xfe : b'\xc3\xbe', # þ + } + + MULTIBYTE_MARKERS_AND_SIZES = [ + (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF + (0xe0, 0xef, 3), # 3-byte characters start with E0-EF + (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4 + ] + + FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0] + LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1] + + @classmethod + def detwingle(cls, in_bytes, main_encoding="utf8", + embedded_encoding="windows-1252"): + """Fix characters from one encoding embedded in some other encoding. + + Currently the only situation supported is Windows-1252 (or its + subset ISO-8859-1), embedded in UTF-8. + + The input must be a bytestring. If you've already converted + the document to Unicode, you're too late. + + The output is a bytestring in which `embedded_encoding` + characters have been converted to their `main_encoding` + equivalents. + """ + if embedded_encoding.replace('_', '-').lower() not in ( + 'windows-1252', 'windows_1252'): + raise NotImplementedError( + "Windows-1252 and ISO-8859-1 are the only currently supported " + "embedded encodings.") + + if main_encoding.lower() not in ('utf8', 'utf-8'): + raise NotImplementedError( + "UTF-8 is the only currently supported main encoding.") + + byte_chunks = [] + + chunk_start = 0 + pos = 0 + while pos < len(in_bytes): + byte = in_bytes[pos] + if not isinstance(byte, int): + # Python 2.x + byte = ord(byte) + if (byte >= cls.FIRST_MULTIBYTE_MARKER + and byte <= cls.LAST_MULTIBYTE_MARKER): + # This is the start of a UTF-8 multibyte character. Skip + # to the end. + for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES: + if byte >= start and byte <= end: + pos += size + break + elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8: + # We found a Windows-1252 character! + # Save the string up to this point as a chunk. + byte_chunks.append(in_bytes[chunk_start:pos]) + + # Now translate the Windows-1252 character into UTF-8 + # and add it as another, one-byte chunk. + byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte]) + pos += 1 + chunk_start = pos + else: + # Go on to the next character. + pos += 1 + if chunk_start == 0: + # The string is unchanged. + return in_bytes + else: + # Store the final chunk. + byte_chunks.append(in_bytes[chunk_start:]) + return b''.join(byte_chunks) + diff --git a/bs4/diagnose.py b/bs4/diagnose.py new file mode 100644 index 0000000..4d0b00a --- /dev/null +++ b/bs4/diagnose.py @@ -0,0 +1,204 @@ +"""Diagnostic functions, mainly for use when doing tech support.""" +import cProfile +from StringIO import StringIO +from HTMLParser import HTMLParser +import bs4 +from bs4 import BeautifulSoup, __version__ +from bs4.builder import builder_registry + +import os +import pstats +import random +import tempfile +import time +import traceback +import sys +import cProfile + +def diagnose(data): + """Diagnostic suite for isolating common problems.""" + print "Diagnostic running on Beautiful Soup %s" % __version__ + print "Python version %s" % sys.version + + basic_parsers = ["html.parser", "html5lib", "lxml"] + for name in basic_parsers: + for builder in builder_registry.builders: + if name in builder.features: + break + else: + basic_parsers.remove(name) + print ( + "I noticed that %s is not installed. Installing it may help." % + name) + + if 'lxml' in basic_parsers: + basic_parsers.append(["lxml", "xml"]) + from lxml import etree + print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)) + + if 'html5lib' in basic_parsers: + import html5lib + print "Found html5lib version %s" % html5lib.__version__ + + if hasattr(data, 'read'): + data = data.read() + elif os.path.exists(data): + print '"%s" looks like a filename. Reading data from the file.' % data + data = open(data).read() + elif data.startswith("http:") or data.startswith("https:"): + print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data + print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup." + return + print + + for parser in basic_parsers: + print "Trying to parse your markup with %s" % parser + success = False + try: + soup = BeautifulSoup(data, parser) + success = True + except Exception, e: + print "%s could not parse the markup." % parser + traceback.print_exc() + if success: + print "Here's what %s did with the markup:" % parser + print soup.prettify() + + print "-" * 80 + +def lxml_trace(data, html=True, **kwargs): + """Print out the lxml events that occur during parsing. + + This lets you see how lxml parses a document when no Beautiful + Soup code is running. + """ + from lxml import etree + for event, element in etree.iterparse(StringIO(data), html=html, **kwargs): + print("%s, %4s, %s" % (event, element.tag, element.text)) + +class AnnouncingParser(HTMLParser): + """Announces HTMLParser parse events, without doing anything else.""" + + def _p(self, s): + print(s) + + def handle_starttag(self, name, attrs): + self._p("%s START" % name) + + def handle_endtag(self, name): + self._p("%s END" % name) + + def handle_data(self, data): + self._p("%s DATA" % data) + + def handle_charref(self, name): + self._p("%s CHARREF" % name) + + def handle_entityref(self, name): + self._p("%s ENTITYREF" % name) + + def handle_comment(self, data): + self._p("%s COMMENT" % data) + + def handle_decl(self, data): + self._p("%s DECL" % data) + + def unknown_decl(self, data): + self._p("%s UNKNOWN-DECL" % data) + + def handle_pi(self, data): + self._p("%s PI" % data) + +def htmlparser_trace(data): + """Print out the HTMLParser events that occur during parsing. + + This lets you see how HTMLParser parses a document when no + Beautiful Soup code is running. + """ + parser = AnnouncingParser() + parser.feed(data) + +_vowels = "aeiou" +_consonants = "bcdfghjklmnpqrstvwxyz" + +def rword(length=5): + "Generate a random word-like string." + s = '' + for i in range(length): + if i % 2 == 0: + t = _consonants + else: + t = _vowels + s += random.choice(t) + return s + +def rsentence(length=4): + "Generate a random sentence-like string." + return " ".join(rword(random.randint(4,9)) for i in range(length)) + +def rdoc(num_elements=1000): + """Randomly generate an invalid HTML document.""" + tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table'] + elements = [] + for i in range(num_elements): + choice = random.randint(0,3) + if choice == 0: + # New tag. + tag_name = random.choice(tag_names) + elements.append("<%s>" % tag_name) + elif choice == 1: + elements.append(rsentence(random.randint(1,4))) + elif choice == 2: + # Close a tag. + tag_name = random.choice(tag_names) + elements.append("" % tag_name) + return "" + "\n".join(elements) + "" + +def benchmark_parsers(num_elements=100000): + """Very basic head-to-head performance benchmark.""" + print "Comparative parser benchmark on Beautiful Soup %s" % __version__ + data = rdoc(num_elements) + print "Generated a large invalid HTML document (%d bytes)." % len(data) + + for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]: + success = False + try: + a = time.time() + soup = BeautifulSoup(data, parser) + b = time.time() + success = True + except Exception, e: + print "%s could not parse the markup." % parser + traceback.print_exc() + if success: + print "BS4+%s parsed the markup in %.2fs." % (parser, b-a) + + from lxml import etree + a = time.time() + etree.HTML(data) + b = time.time() + print "Raw lxml parsed the markup in %.2fs." % (b-a) + + import html5lib + parser = html5lib.HTMLParser() + a = time.time() + parser.parse(data) + b = time.time() + print "Raw html5lib parsed the markup in %.2fs." % (b-a) + +def profile(num_elements=100000, parser="lxml"): + + filehandle = tempfile.NamedTemporaryFile() + filename = filehandle.name + + data = rdoc(num_elements) + vars = dict(bs4=bs4, data=data, parser=parser) + cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename) + + stats = pstats.Stats(filename) + # stats.strip_dirs() + stats.sort_stats("cumulative") + stats.print_stats('_html5lib|bs4', 50) + +if __name__ == '__main__': + diagnose(sys.stdin.read()) diff --git a/bs4/element.py b/bs4/element.py new file mode 100644 index 0000000..da9afdf --- /dev/null +++ b/bs4/element.py @@ -0,0 +1,1611 @@ +import collections +import re +import sys +import warnings +from bs4.dammit import EntitySubstitution + +DEFAULT_OUTPUT_ENCODING = "utf-8" +PY3K = (sys.version_info[0] > 2) + +whitespace_re = re.compile("\s+") + +def _alias(attr): + """Alias one attribute name to another for backward compatibility""" + @property + def alias(self): + return getattr(self, attr) + + @alias.setter + def alias(self): + return setattr(self, attr) + return alias + + +class NamespacedAttribute(unicode): + + def __new__(cls, prefix, name, namespace=None): + if name is None: + obj = unicode.__new__(cls, prefix) + elif prefix is None: + # Not really namespaced. + obj = unicode.__new__(cls, name) + else: + obj = unicode.__new__(cls, prefix + ":" + name) + obj.prefix = prefix + obj.name = name + obj.namespace = namespace + return obj + +class AttributeValueWithCharsetSubstitution(unicode): + """A stand-in object for a character encoding specified in HTML.""" + +class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution): + """A generic stand-in for the value of a meta tag's 'charset' attribute. + + When Beautiful Soup parses the markup '', the + value of the 'charset' attribute will be one of these objects. + """ + + def __new__(cls, original_value): + obj = unicode.__new__(cls, original_value) + obj.original_value = original_value + return obj + + def encode(self, encoding): + return encoding + + +class ContentMetaAttributeValue(AttributeValueWithCharsetSubstitution): + """A generic stand-in for the value of a meta tag's 'content' attribute. + + When Beautiful Soup parses the markup: + + + The value of the 'content' attribute will be one of these objects. + """ + + CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M) + + def __new__(cls, original_value): + match = cls.CHARSET_RE.search(original_value) + if match is None: + # No substitution necessary. + return unicode.__new__(unicode, original_value) + + obj = unicode.__new__(cls, original_value) + obj.original_value = original_value + return obj + + def encode(self, encoding): + def rewrite(match): + return match.group(1) + encoding + return self.CHARSET_RE.sub(rewrite, self.original_value) + +class HTMLAwareEntitySubstitution(EntitySubstitution): + + """Entity substitution rules that are aware of some HTML quirks. + + Specifically, the contents of + + +
+

+ FanFictionDownLoader +

+ +
+ + +
+ +
+ +
+

Edit Config

+
+ Editing configuration for {{ nickname }}. +
+
+ +
+
+ +
+ +
+
+ +
+

Default System configuration

+
+{{ defaultsini }}
+
+
+ +
+ Powered by Google App Engine +

+ This is a web front-end to FanFictionDownLoader
+ Copyright © Fanficdownloader team +
+ +
+ + +
+
+ + diff --git a/epubmerge.py b/epubmerge.py new file mode 100644 index 0000000..f7e76b8 --- /dev/null +++ b/epubmerge.py @@ -0,0 +1,25 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# epubmerge.py 1.0 + +# Copyright 2011, Jim Miller + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if __name__ == "__main__": + print(''' +The this utility has been split out into it's own project. +See: http://code.google.com/p/epubmerge/ +...for a CLI epubmerge.py program and calibre plugin. +''') diff --git a/example.ini b/example.ini new file mode 100644 index 0000000..942cd60 --- /dev/null +++ b/example.ini @@ -0,0 +1,103 @@ +## This is an example of what your personal configuration might look +## like. Uncomment options by removing the '#' in front of them. + +[defaults] +## Some sites also require the user to confirm they are adult for +## adult content. Uncomment by removing '#' in front of is_adult. In +## commandline version, this should go in your personal.ini, not +## defaults.ini. +#is_adult:true + +## Don't like the numbers at the start of chapter titles on some +## sites? You can use strip_chapter_numbers to strip them off. Just +## want to make them all look the same? Strip them off, then add them +## back on with add_chapter_numbers. Don't like the way it strips +## numbers or adds them back? See chapter_title_strip_pattern and +## chapter_title_add_pattern. +#strip_chapter_numbers:true +#add_chapter_numbers:true + +[epub] +## include images from img tags in the body and summary of stories. +## Images will be converted to jpg for size if possible. Images work +## in epub format only. To get mobi or other format with images, +## download as epub and use Calibre to convert. +#include_images:true + +## If not set, the summary will have all html stripped for safety. +## Both this and include_images must be true to get images in the +## summary. +#keep_summary_html:true + +## If set, the first image found will be made the cover image. If +## keep_summary_html is true, any images in summary will be before any +## in chapters. +#make_firstimage_cover:true + +## Resize images down to width, height, preserving aspect ratio. +## Nook size, with margin. +#image_max_size: 580, 725 + +## Change image to grayscale, if graphics library allows, to save +## space. +#grayscale_images: false + + +## Most common, I expect will be using this to save username/passwords +## for different sites. Here are a few examples. See defaults.ini +## for the full list. + +[www.twilighted.net] +#username:YourPenname +#password:YourPassword +## default is false +#collect_series: true + +[ficwad.com] +#username:YourUsername +#password:YourPassword + +[www.twiwrite.net] +#username:YourName +#password:yourpassword +## default is false +#collect_series: true + +[www.adastrafanfic.com] +## Some sites do not require a login, but do require the user to +## confirm they are adult for adult content. +#is_adult:true + +[www.twcslibrary.net] +#username:YourName +#password:yourpassword +#is_adult:true +## default is false +#collect_series: true + +[www.fictionalley.org] +#is_adult:true + +[www.harrypotterfanfiction.com] +#is_adult:true + +[www.fimfiction.net] +#is_adult:true +#fail_on_password: false + +[www.tthfanfic.org] +#is_adult:true +## tth is a little unusual--it doesn't require user/pass, but the site +## keeps track of which chapters you've read and won't send another +## update until it thinks you're up to date. This way, on download, +## it thinks you're up to date. +#username:YourName +#password:yourpassword + + +## This section will override anything in the system defaults or other +## sections here. +[overrides] +## default varies by site. Set true here to force all sites to +## collect series. +#collect_series: true diff --git a/fanficdownloader.zip b/fanficdownloader.zip new file mode 100644 index 0000000..af01480 Binary files /dev/null and b/fanficdownloader.zip differ diff --git a/fanficdownloader/BeautifulSoup.py b/fanficdownloader/BeautifulSoup.py new file mode 100644 index 0000000..8afb6bf --- /dev/null +++ b/fanficdownloader/BeautifulSoup.py @@ -0,0 +1,2017 @@ +"""Beautiful Soup +Elixir and Tonic +"The Screen-Scraper's Friend" +http://www.crummy.com/software/BeautifulSoup/ + +Beautiful Soup parses a (possibly invalid) XML or HTML document into a +tree representation. It provides methods and Pythonic idioms that make +it easy to navigate, search, and modify the tree. + +A well-formed XML/HTML document yields a well-formed data +structure. An ill-formed XML/HTML document yields a correspondingly +ill-formed data structure. If your document is only locally +well-formed, you can use this library to find and process the +well-formed part of it. + +Beautiful Soup works with Python 2.2 and up. It has no external +dependencies, but you'll have more success at converting data to UTF-8 +if you also install these three packages: + +* chardet, for auto-detecting character encodings + http://chardet.feedparser.org/ +* cjkcodecs and iconv_codec, which add more encodings to the ones supported + by stock Python. + http://cjkpython.i18n.org/ + +Beautiful Soup defines classes for two main parsing strategies: + + * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific + language that kind of looks like XML. + + * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid + or invalid. This class has web browser-like heuristics for + obtaining a sensible parse tree in the face of common HTML errors. + +Beautiful Soup also defines a class (UnicodeDammit) for autodetecting +the encoding of an HTML or XML document, and converting it to +Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed Parser. + +For more than you ever wanted to know about Beautiful Soup, see the +documentation: +http://www.crummy.com/software/BeautifulSoup/documentation.html + +Here, have some legalese: + +Copyright (c) 2004-2010, Leonard Richardson + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the the Beautiful Soup Consortium and All + Night Kosher Bakery nor the names of its contributors may be + used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT. + +""" +from __future__ import generators + +__author__ = "Leonard Richardson (leonardr@segfault.org)" +__version__ = "3.2.0" +__copyright__ = "Copyright (c) 2004-2010 Leonard Richardson" +__license__ = "New-style BSD" + +from sgmllib import SGMLParser, SGMLParseError +import codecs +import markupbase +import types +import re +import sgmllib +try: + from htmlentitydefs import name2codepoint +except ImportError: + name2codepoint = {} +try: + set +except NameError: + from sets import Set as set + +#These hacks make Beautiful Soup able to parse XML with namespaces +sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*') +markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match + +DEFAULT_OUTPUT_ENCODING = "utf-8" + +def _match_css_class(str): + """Build a RE to match the given CSS class.""" + return re.compile(r"(^|.*\s)%s($|\s)" % str) + +# First, the classes that represent markup elements. + +class PageElement(object): + """Contains the navigational information for some part of the page + (either a tag or a piece of text)""" + + ## Added for FFDL. + bs3=True + + def setup(self, parent=None, previous=None): + """Sets up the initial relations between this element and + other elements.""" + self.parent = parent + self.previous = previous + self.next = None + self.previousSibling = None + self.nextSibling = None + if self.parent and self.parent.contents: + self.previousSibling = self.parent.contents[-1] + self.previousSibling.nextSibling = self + + def replaceWith(self, replaceWith): + oldParent = self.parent + myIndex = self.parent.index(self) + if hasattr(replaceWith, "parent")\ + and replaceWith.parent is self.parent: + # We're replacing this element with one of its siblings. + index = replaceWith.parent.index(replaceWith) + if index and index < myIndex: + # Furthermore, it comes before this element. That + # means that when we extract it, the index of this + # element will change. + myIndex = myIndex - 1 + self.extract() + oldParent.insert(myIndex, replaceWith) + + def replaceWithChildren(self): + myParent = self.parent + myIndex = self.parent.index(self) + self.extract() + reversedChildren = list(self.contents) + reversedChildren.reverse() + for child in reversedChildren: + myParent.insert(myIndex, child) + + def extract(self): + """Destructively rips this element out of the tree.""" + if self.parent: + try: + del self.parent.contents[self.parent.index(self)] + except ValueError: + pass + + #Find the two elements that would be next to each other if + #this element (and any children) hadn't been parsed. Connect + #the two. + lastChild = self._lastRecursiveChild() + nextElement = lastChild.next + + if self.previous: + self.previous.next = nextElement + if nextElement: + nextElement.previous = self.previous + self.previous = None + lastChild.next = None + + self.parent = None + if self.previousSibling: + self.previousSibling.nextSibling = self.nextSibling + if self.nextSibling: + self.nextSibling.previousSibling = self.previousSibling + self.previousSibling = self.nextSibling = None + return self + + def _lastRecursiveChild(self): + "Finds the last element beneath this object to be parsed." + lastChild = self + while hasattr(lastChild, 'contents') and lastChild.contents: + lastChild = lastChild.contents[-1] + return lastChild + + def insert(self, position, newChild): + if isinstance(newChild, basestring) \ + and not isinstance(newChild, NavigableString): + newChild = NavigableString(newChild) + + position = min(position, len(self.contents)) + if hasattr(newChild, 'parent') and newChild.parent is not None: + # We're 'inserting' an element that's already one + # of this object's children. + if newChild.parent is self: + index = self.index(newChild) + if index > position: + # Furthermore we're moving it further down the + # list of this object's children. That means that + # when we extract this element, our target index + # will jump down one. + position = position - 1 + newChild.extract() + + newChild.parent = self + previousChild = None + if position == 0: + newChild.previousSibling = None + newChild.previous = self + else: + previousChild = self.contents[position-1] + newChild.previousSibling = previousChild + newChild.previousSibling.nextSibling = newChild + newChild.previous = previousChild._lastRecursiveChild() + if newChild.previous: + newChild.previous.next = newChild + + newChildsLastElement = newChild._lastRecursiveChild() + + if position >= len(self.contents): + newChild.nextSibling = None + + parent = self + parentsNextSibling = None + while not parentsNextSibling: + parentsNextSibling = parent.nextSibling + parent = parent.parent + if not parent: # This is the last element in the document. + break + if parentsNextSibling: + newChildsLastElement.next = parentsNextSibling + else: + newChildsLastElement.next = None + else: + nextChild = self.contents[position] + newChild.nextSibling = nextChild + if newChild.nextSibling: + newChild.nextSibling.previousSibling = newChild + newChildsLastElement.next = nextChild + + if newChildsLastElement.next: + newChildsLastElement.next.previous = newChildsLastElement + self.contents.insert(position, newChild) + + def append(self, tag): + """Appends the given tag to the contents of this tag.""" + self.insert(len(self.contents), tag) + + def findNext(self, name=None, attrs={}, text=None, **kwargs): + """Returns the first item that matches the given criteria and + appears after this Tag in the document.""" + return self._findOne(self.findAllNext, name, attrs, text, **kwargs) + + def findAllNext(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns all items that match the given criteria and appear + after this Tag in the document.""" + return self._findAll(name, attrs, text, limit, self.nextGenerator, + **kwargs) + + def findNextSibling(self, name=None, attrs={}, text=None, **kwargs): + """Returns the closest sibling to this Tag that matches the + given criteria and appears after this Tag in the document.""" + return self._findOne(self.findNextSiblings, name, attrs, text, + **kwargs) + + def findNextSiblings(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns the siblings of this Tag that match the given + criteria and appear after this Tag in the document.""" + return self._findAll(name, attrs, text, limit, + self.nextSiblingGenerator, **kwargs) + fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x + + def findPrevious(self, name=None, attrs={}, text=None, **kwargs): + """Returns the first item that matches the given criteria and + appears before this Tag in the document.""" + return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs) + + def findAllPrevious(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns all items that match the given criteria and appear + before this Tag in the document.""" + return self._findAll(name, attrs, text, limit, self.previousGenerator, + **kwargs) + fetchPrevious = findAllPrevious # Compatibility with pre-3.x + + def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs): + """Returns the closest sibling to this Tag that matches the + given criteria and appears before this Tag in the document.""" + return self._findOne(self.findPreviousSiblings, name, attrs, text, + **kwargs) + + def findPreviousSiblings(self, name=None, attrs={}, text=None, + limit=None, **kwargs): + """Returns the siblings of this Tag that match the given + criteria and appear before this Tag in the document.""" + return self._findAll(name, attrs, text, limit, + self.previousSiblingGenerator, **kwargs) + fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x + + def findParent(self, name=None, attrs={}, **kwargs): + """Returns the closest parent of this Tag that matches the given + criteria.""" + # NOTE: We can't use _findOne because findParents takes a different + # set of arguments. + r = None + l = self.findParents(name, attrs, 1) + if l: + r = l[0] + return r + + def findParents(self, name=None, attrs={}, limit=None, **kwargs): + """Returns the parents of this Tag that match the given + criteria.""" + + return self._findAll(name, attrs, None, limit, self.parentGenerator, + **kwargs) + fetchParents = findParents # Compatibility with pre-3.x + + #These methods do the real heavy lifting. + + def _findOne(self, method, name, attrs, text, **kwargs): + r = None + l = method(name, attrs, text, 1, **kwargs) + if l: + r = l[0] + return r + + def _findAll(self, name, attrs, text, limit, generator, **kwargs): + "Iterates over a generator looking for things that match." + + if isinstance(name, SoupStrainer): + strainer = name + # (Possibly) special case some findAll*(...) searches + elif text is None and not limit and not attrs and not kwargs: + # findAll*(True) + if name is True: + return [element for element in generator() + if isinstance(element, Tag)] + # findAll*('tag-name') + elif isinstance(name, basestring): + return [element for element in generator() + if isinstance(element, Tag) and + element.name == name] + else: + strainer = SoupStrainer(name, attrs, text, **kwargs) + # Build a SoupStrainer + else: + strainer = SoupStrainer(name, attrs, text, **kwargs) + results = ResultSet(strainer) + g = generator() + while True: + try: + i = g.next() + except StopIteration: + break + if i: + found = strainer.search(i) + if found: + results.append(found) + if limit and len(results) >= limit: + break + return results + + #These Generators can be used to navigate starting from both + #NavigableStrings and Tags. + def nextGenerator(self): + i = self + while i is not None: + i = i.next + yield i + + def nextSiblingGenerator(self): + i = self + while i is not None: + i = i.nextSibling + yield i + + def previousGenerator(self): + i = self + while i is not None: + i = i.previous + yield i + + def previousSiblingGenerator(self): + i = self + while i is not None: + i = i.previousSibling + yield i + + def parentGenerator(self): + i = self + while i is not None: + i = i.parent + yield i + + # Utility methods + def substituteEncoding(self, str, encoding=None): + encoding = encoding or "utf-8" + return str.replace("%SOUP-ENCODING%", encoding) + + def toEncoding(self, s, encoding=None): + """Encodes an object to a string in some encoding, or to Unicode. + .""" + if isinstance(s, unicode): + if encoding: + s = s.encode(encoding) + elif isinstance(s, str): + if encoding: + s = s.encode(encoding) + else: + s = unicode(s) + else: + if encoding: + s = self.toEncoding(str(s), encoding) + else: + s = unicode(s) + return s + +class NavigableString(unicode, PageElement): + + def __new__(cls, value): + """Create a new NavigableString. + + When unpickling a NavigableString, this method is called with + the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be + passed in to the superclass's __new__ or the superclass won't know + how to handle non-ASCII characters. + """ + if isinstance(value, unicode): + return unicode.__new__(cls, value) + return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING) + + def __getnewargs__(self): + return (NavigableString.__str__(self),) + + def __getattr__(self, attr): + """text.string gives you text. This is for backwards + compatibility for Navigable*String, but for CData* it lets you + get the string without the CData wrapper.""" + if attr == 'string': + return self + else: + raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr) + + def __unicode__(self): + return str(self).decode(DEFAULT_OUTPUT_ENCODING) + + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + if encoding: + return self.encode(encoding) + else: + return self + +class CData(NavigableString): + + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + return "" % NavigableString.__str__(self, encoding) + +class ProcessingInstruction(NavigableString): + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + output = self + if "%SOUP-ENCODING%" in output: + output = self.substituteEncoding(output, encoding) + return "" % self.toEncoding(output, encoding) + +class Comment(NavigableString): + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + return "" % NavigableString.__str__(self, encoding) + +class Declaration(NavigableString): + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + return "" % NavigableString.__str__(self, encoding) + +class Tag(PageElement): + + """Represents a found HTML tag with its attributes and contents.""" + + def _invert(h): + "Cheap function to invert a hash." + i = {} + for k,v in h.items(): + i[v] = k + return i + + XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'", + "quot" : '"', + "amp" : "&", + "lt" : "<", + "gt" : ">" } + + XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS) + + def _convertEntities(self, match): + """Used in a call to re.sub to replace HTML, XML, and numeric + entities with the appropriate Unicode characters. If HTML + entities are being converted, any unrecognized entities are + escaped.""" + x = match.group(1) + if self.convertHTMLEntities and x in name2codepoint: + return unichr(name2codepoint[x]) + elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS: + if self.convertXMLEntities: + return self.XML_ENTITIES_TO_SPECIAL_CHARS[x] + else: + return u'&%s;' % x + elif len(x) > 0 and x[0] == '#': + # Handle numeric entities + if len(x) > 1 and x[1] == 'x': + return unichr(int(x[2:], 16)) + else: + return unichr(int(x[1:])) + + elif self.escapeUnrecognizedEntities: + return u'&%s;' % x + else: + return u'&%s;' % x + + def __init__(self, parser, name, attrs=None, parent=None, + previous=None): + "Basic constructor." + + # We don't actually store the parser object: that lets extracted + # chunks be garbage-collected + self.parserClass = parser.__class__ + self.isSelfClosing = parser.isSelfClosingTag(name) + self.name = name + if attrs is None: + attrs = [] + elif isinstance(attrs, dict): + attrs = attrs.items() + self.attrs = attrs + self.contents = [] + self.setup(parent, previous) + self.hidden = False + self.containsSubstitutions = False + self.convertHTMLEntities = parser.convertHTMLEntities + self.convertXMLEntities = parser.convertXMLEntities + self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities + + # Convert any HTML, XML, or numeric entities in the attribute values. + convert = lambda(k, val): (k, + re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);", + self._convertEntities, + val)) + self.attrs = map(convert, self.attrs) + + def getString(self): + if (len(self.contents) == 1 + and isinstance(self.contents[0], NavigableString)): + return self.contents[0] + + def setString(self, string): + """Replace the contents of the tag with a string""" + self.clear() + self.append(string) + + string = property(getString, setString) + + def getText(self, separator=u""): + if not len(self.contents): + return u"" + stopNode = self._lastRecursiveChild().next + strings = [] + current = self.contents[0] + while current is not stopNode: + if isinstance(current, NavigableString): + strings.append(current.strip()) + current = current.next + return separator.join(strings) + + text = property(getText) + + def get(self, key, default=None): + """Returns the value of the 'key' attribute for the tag, or + the value given for 'default' if it doesn't have that + attribute.""" + return self._getAttrMap().get(key, default) + + def clear(self): + """Extract all children.""" + for child in self.contents[:]: + child.extract() + + def index(self, element): + for i, child in enumerate(self.contents): + if child is element: + return i + raise ValueError("Tag.index: element not in tag") + + def has_key(self, key): + return self._getAttrMap().has_key(key) + + def __getitem__(self, key): + """tag[key] returns the value of the 'key' attribute for the tag, + and throws an exception if it's not there.""" + return self._getAttrMap()[key] + + def __iter__(self): + "Iterating over a tag iterates over its contents." + return iter(self.contents) + + def __len__(self): + "The length of a tag is the length of its list of contents." + return len(self.contents) + + def __contains__(self, x): + return x in self.contents + + def __nonzero__(self): + "A tag is non-None even if it has no contents." + return True + + def __setitem__(self, key, value): + """Setting tag[key] sets the value of the 'key' attribute for the + tag.""" + self._getAttrMap() + self.attrMap[key] = value + found = False + for i in range(0, len(self.attrs)): + if self.attrs[i][0] == key: + self.attrs[i] = (key, value) + found = True + if not found: + self.attrs.append((key, value)) + self._getAttrMap()[key] = value + + def __delitem__(self, key): + "Deleting tag[key] deletes all 'key' attributes for the tag." + for item in self.attrs: + if item[0] == key: + self.attrs.remove(item) + #We don't break because bad HTML can define the same + #attribute multiple times. + self._getAttrMap() + if self.attrMap.has_key(key): + del self.attrMap[key] + + def __call__(self, *args, **kwargs): + """Calling a tag like a function is the same as calling its + findAll() method. Eg. tag('a') returns a list of all the A tags + found within this tag.""" + return apply(self.findAll, args, kwargs) + + def __getattr__(self, tag): + #print "Getattr %s.%s" % (self.__class__, tag) + if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3: + return self.find(tag[:-3]) + elif tag.find('__') != 0: + return self.find(tag) + raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag) + + def __eq__(self, other): + """Returns true iff this tag has the same name, the same attributes, + and the same contents (recursively) as the given tag. + + NOTE: right now this will return false if two tags have the + same attributes in a different order. Should this be fixed?""" + if other is self: + return True + if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other): + return False + for i in range(0, len(self.contents)): + if self.contents[i] != other.contents[i]: + return False + return True + + def __ne__(self, other): + """Returns true iff this tag is not identical to the other tag, + as defined in __eq__.""" + return not self == other + + def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING): + """Renders this tag as a string.""" + return self.__str__(encoding) + + def __unicode__(self): + return str(self).decode(DEFAULT_OUTPUT_ENCODING) + + BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" + + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" + + ")") + + def _sub_entity(self, x): + """Used with a regular expression to substitute the + appropriate XML entity for an XML special character.""" + return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";" + + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING, + prettyPrint=False, indentLevel=0): + """Returns a string or Unicode representation of this tag and + its contents. To get Unicode, pass None for encoding. + + NOTE: since Python's HTML parser consumes whitespace, this + method is not certain to reproduce the whitespace present in + the original string.""" + + encodedName = self.toEncoding(self.name, encoding) + + attrs = [] + if self.attrs: + for key, val in self.attrs: + fmt = '%s="%s"' + if isinstance(val, basestring): + if self.containsSubstitutions and '%SOUP-ENCODING%' in val: + val = self.substituteEncoding(val, encoding) + + # The attribute value either: + # + # * Contains no embedded double quotes or single quotes. + # No problem: we enclose it in double quotes. + # * Contains embedded single quotes. No problem: + # double quotes work here too. + # * Contains embedded double quotes. No problem: + # we enclose it in single quotes. + # * Embeds both single _and_ double quotes. This + # can't happen naturally, but it can happen if + # you modify an attribute value after parsing + # the document. Now we have a bit of a + # problem. We solve it by enclosing the + # attribute in single quotes, and escaping any + # embedded single quotes to XML entities. + if '"' in val: + fmt = "%s='%s'" + if "'" in val: + # TODO: replace with apos when + # appropriate. + val = val.replace("'", "&squot;") + + # Now we're okay w/r/t quotes. But the attribute + # value might also contain angle brackets, or + # ampersands that aren't part of entities. We need + # to escape those to XML entities too. + val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val) + + attrs.append(fmt % (self.toEncoding(key, encoding), + self.toEncoding(val, encoding))) + close = '' + closeTag = '' + if self.isSelfClosing: + close = ' /' + else: + closeTag = '' % encodedName + + indentTag, indentContents = 0, 0 + if prettyPrint: + indentTag = indentLevel + space = (' ' * (indentTag-1)) + indentContents = indentTag + 1 + contents = self.renderContents(encoding, prettyPrint, indentContents) + if self.hidden: + s = contents + else: + s = [] + attributeString = '' + if attrs: + attributeString = ' ' + ' '.join(attrs) + if prettyPrint: + s.append(space) + s.append('<%s%s%s>' % (encodedName, attributeString, close)) + if prettyPrint: + s.append("\n") + s.append(contents) + if prettyPrint and contents and contents[-1] != "\n": + s.append("\n") + if prettyPrint and closeTag: + s.append(space) + s.append(closeTag) + if prettyPrint and closeTag and self.nextSibling: + s.append("\n") + s = ''.join(s) + return s + + def decompose(self): + """Recursively destroys the contents of this tree.""" + self.extract() + if len(self.contents) == 0: + return + current = self.contents[0] + while current is not None: + next = current.next + if isinstance(current, Tag): + del current.contents[:] + current.parent = None + current.previous = None + current.previousSibling = None + current.next = None + current.nextSibling = None + current = next + + def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING): + return self.__str__(encoding, True) + + def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING, + prettyPrint=False, indentLevel=0): + """Renders the contents of this tag as a string in the given + encoding. If encoding is None, returns a Unicode string..""" + s=[] + for c in self: + text = None + if isinstance(c, NavigableString): + text = c.__str__(encoding) + elif isinstance(c, Tag): + s.append(c.__str__(encoding, prettyPrint, indentLevel)) + if text and prettyPrint: + text = text.strip() + if text: + if prettyPrint: + s.append(" " * (indentLevel-1)) + s.append(text) + if prettyPrint: + s.append("\n") + return ''.join(s) + + #Soup methods + + def find(self, name=None, attrs={}, recursive=True, text=None, + **kwargs): + """Return only the first child of this Tag matching the given + criteria.""" + r = None + l = self.findAll(name, attrs, recursive, text, 1, **kwargs) + if l: + r = l[0] + return r + findChild = find + + def findAll(self, name=None, attrs={}, recursive=True, text=None, + limit=None, **kwargs): + """Extracts a list of Tag objects that match the given + criteria. You can specify the name of the Tag and any + attributes you want the Tag to have. + + The value of a key-value pair in the 'attrs' map can be a + string, a list of strings, a regular expression object, or a + callable that takes a string and returns whether or not the + string matches for some custom definition of 'matches'. The + same is true of the tag name.""" + generator = self.recursiveChildGenerator + if not recursive: + generator = self.childGenerator + return self._findAll(name, attrs, text, limit, generator, **kwargs) + findChildren = findAll + + # Pre-3.x compatibility methods + first = find + fetch = findAll + + def fetchText(self, text=None, recursive=True, limit=None): + return self.findAll(text=text, recursive=recursive, limit=limit) + + def firstText(self, text=None, recursive=True): + return self.find(text=text, recursive=recursive) + + #Private methods + + def _getAttrMap(self): + """Initializes a map representation of this tag's attributes, + if not already initialized.""" + if not getattr(self, 'attrMap'): + self.attrMap = {} + for (key, value) in self.attrs: + self.attrMap[key] = value + return self.attrMap + + #Generator methods + def childGenerator(self): + # Just use the iterator from the contents + return iter(self.contents) + + def recursiveChildGenerator(self): + if not len(self.contents): + raise StopIteration + stopNode = self._lastRecursiveChild().next + current = self.contents[0] + while current is not stopNode: + yield current + current = current.next + + +# Next, a couple classes to represent queries and their results. +class SoupStrainer: + """Encapsulates a number of ways of matching a markup element (tag or + text).""" + + def __init__(self, name=None, attrs={}, text=None, **kwargs): + self.name = name + if isinstance(attrs, basestring): + kwargs['class'] = _match_css_class(attrs) + attrs = None + if kwargs: + if attrs: + attrs = attrs.copy() + attrs.update(kwargs) + else: + attrs = kwargs + self.attrs = attrs + self.text = text + + def __str__(self): + if self.text: + return self.text + else: + return "%s|%s" % (self.name, self.attrs) + + def searchTag(self, markupName=None, markupAttrs={}): + found = None + markup = None + if isinstance(markupName, Tag): + markup = markupName + markupAttrs = markup + callFunctionWithTagData = callable(self.name) \ + and not isinstance(markupName, Tag) + + if (not self.name) \ + or callFunctionWithTagData \ + or (markup and self._matches(markup, self.name)) \ + or (not markup and self._matches(markupName, self.name)): + if callFunctionWithTagData: + match = self.name(markupName, markupAttrs) + else: + match = True + markupAttrMap = None + for attr, matchAgainst in self.attrs.items(): + if not markupAttrMap: + if hasattr(markupAttrs, 'get'): + markupAttrMap = markupAttrs + else: + markupAttrMap = {} + for k,v in markupAttrs: + markupAttrMap[k] = v + attrValue = markupAttrMap.get(attr) + if not self._matches(attrValue, matchAgainst): + match = False + break + if match: + if markup: + found = markup + else: + found = markupName + return found + + def search(self, markup): + #print 'looking for %s in %s' % (self, markup) + found = None + # If given a list of items, scan it for a text element that + # matches. + if hasattr(markup, "__iter__") \ + and not isinstance(markup, Tag): + for element in markup: + if isinstance(element, NavigableString) \ + and self.search(element): + found = element + break + # If it's a Tag, make sure its name or attributes match. + # Don't bother with Tags if we're searching for text. + elif isinstance(markup, Tag): + if not self.text: + found = self.searchTag(markup) + # If it's text, make sure the text matches. + elif isinstance(markup, NavigableString) or \ + isinstance(markup, basestring): + if self._matches(markup, self.text): + found = markup + else: + raise Exception, "I don't know how to match against a %s" \ + % markup.__class__ + return found + + def _matches(self, markup, matchAgainst): + #print "Matching %s against %s" % (markup, matchAgainst) + result = False + if matchAgainst is True: + result = markup is not None + elif callable(matchAgainst): + result = matchAgainst(markup) + else: + #Custom match methods take the tag as an argument, but all + #other ways of matching match the tag name as a string. + if isinstance(markup, Tag): + markup = markup.name + if markup and not isinstance(markup, basestring): + markup = unicode(markup) + #Now we know that chunk is either a string, or None. + if hasattr(matchAgainst, 'match'): + # It's a regexp object. + result = markup and matchAgainst.search(markup) + elif hasattr(matchAgainst, '__iter__'): # list-like + result = markup in matchAgainst + elif hasattr(matchAgainst, 'items'): + result = markup.has_key(matchAgainst) + elif matchAgainst and isinstance(markup, basestring): + if isinstance(markup, unicode): + matchAgainst = unicode(matchAgainst) + else: + matchAgainst = str(matchAgainst) + + if not result: + result = matchAgainst == markup + return result + +class ResultSet(list): + """A ResultSet is just a list that keeps track of the SoupStrainer + that created it.""" + def __init__(self, source): + list.__init__([]) + self.source = source + +# Now, some helper functions. + +def buildTagMap(default, *args): + """Turns a list of maps, lists, or scalars into a single map. + Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and + NESTING_RESET_TAGS maps out of lists and partial maps.""" + built = {} + for portion in args: + if hasattr(portion, 'items'): + #It's a map. Merge it. + for k,v in portion.items(): + built[k] = v + elif hasattr(portion, '__iter__'): # is a list + #It's a list. Map each item to the default. + for k in portion: + built[k] = default + else: + #It's a scalar. Map it to the default. + built[portion] = default + return built + +# Now, the parser classes. + +class BeautifulStoneSoup(Tag, SGMLParser): + + """This class contains the basic parser and search code. It defines + a parser that knows nothing about tag behavior except for the + following: + + You can't close a tag without closing all the tags it encloses. + That is, "" actually means + "". + + [Another possible explanation is "", but since + this class defines no SELF_CLOSING_TAGS, it will never use that + explanation.] + + This class is useful for parsing XML or made-up markup languages, + or when BeautifulSoup makes an assumption counter to what you were + expecting.""" + + SELF_CLOSING_TAGS = {} + NESTABLE_TAGS = {} + RESET_NESTING_TAGS = {} + QUOTE_TAGS = {} + PRESERVE_WHITESPACE_TAGS = [] + + MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'), + lambda x: x.group(1) + ' />'), + (re.compile(']*)>'), + lambda x: '') + ] + + ROOT_TAG_NAME = u'[document]' + + HTML_ENTITIES = "html" + XML_ENTITIES = "xml" + XHTML_ENTITIES = "xhtml" + # TODO: This only exists for backwards-compatibility + ALL_ENTITIES = XHTML_ENTITIES + + # Used when determining whether a text node is all whitespace and + # can be replaced with a single space. A text node that contains + # fancy Unicode spaces (usually non-breaking) should be left + # alone. + STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, } + + def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None, + markupMassage=True, smartQuotesTo=XML_ENTITIES, + convertEntities=None, selfClosingTags=None, isHTML=False): + """The Soup object is initialized as the 'root tag', and the + provided markup (which can be a string or a file-like object) + is fed into the underlying parser. + + sgmllib will process most bad HTML, and the BeautifulSoup + class has some tricks for dealing with some HTML that kills + sgmllib, but Beautiful Soup can nonetheless choke or lose data + if your data uses self-closing tags or declarations + incorrectly. + + By default, Beautiful Soup uses regexes to sanitize input, + avoiding the vast majority of these problems. If the problems + don't apply to you, pass in False for markupMassage, and + you'll get better performance. + + The default parser massage techniques fix the two most common + instances of invalid HTML that choke sgmllib: + +
(No space between name of closing tag and tag close) + (Extraneous whitespace in declaration) + + You can pass in a custom list of (RE object, replace method) + tuples to get Beautiful Soup to scrub your input the way you + want.""" + + self.parseOnlyThese = parseOnlyThese + self.fromEncoding = fromEncoding + self.smartQuotesTo = smartQuotesTo + self.convertEntities = convertEntities + # Set the rules for how we'll deal with the entities we + # encounter + if self.convertEntities: + # It doesn't make sense to convert encoded characters to + # entities even while you're converting entities to Unicode. + # Just convert it all to Unicode. + self.smartQuotesTo = None + if convertEntities == self.HTML_ENTITIES: + self.convertXMLEntities = False + self.convertHTMLEntities = True + self.escapeUnrecognizedEntities = True + elif convertEntities == self.XHTML_ENTITIES: + self.convertXMLEntities = True + self.convertHTMLEntities = True + self.escapeUnrecognizedEntities = False + elif convertEntities == self.XML_ENTITIES: + self.convertXMLEntities = True + self.convertHTMLEntities = False + self.escapeUnrecognizedEntities = False + else: + self.convertXMLEntities = False + self.convertHTMLEntities = False + self.escapeUnrecognizedEntities = False + + self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags) + SGMLParser.__init__(self) + + if hasattr(markup, 'read'): # It's a file-type object. + markup = markup.read() + self.markup = markup + self.markupMassage = markupMassage + try: + self._feed(isHTML=isHTML) + except StopParsing: + pass + self.markup = None # The markup can now be GCed + + def convert_charref(self, name): + """This method fixes a bug in Python's SGMLParser.""" + try: + n = int(name) + except ValueError: + return + if not 0 <= n <= 127 : # ASCII ends at 127, not 255 + return + return self.convert_codepoint(n) + + def _feed(self, inDocumentEncoding=None, isHTML=False): + # Convert the document to Unicode. + markup = self.markup + if isinstance(markup, unicode): + if not hasattr(self, 'originalEncoding'): + self.originalEncoding = None + else: + dammit = UnicodeDammit\ + (markup, [self.fromEncoding, inDocumentEncoding], + smartQuotesTo=self.smartQuotesTo, isHTML=isHTML) + markup = dammit.unicode + self.originalEncoding = dammit.originalEncoding + self.declaredHTMLEncoding = dammit.declaredHTMLEncoding + if markup: + if self.markupMassage: + if not hasattr(self.markupMassage, "__iter__"): + self.markupMassage = self.MARKUP_MASSAGE + for fix, m in self.markupMassage: + markup = fix.sub(m, markup) + # TODO: We get rid of markupMassage so that the + # soup object can be deepcopied later on. Some + # Python installations can't copy regexes. If anyone + # was relying on the existence of markupMassage, this + # might cause problems. + del(self.markupMassage) + self.reset() + + SGMLParser.feed(self, markup) + # Close out any unfinished strings and close all the open tags. + self.endData() + while self.currentTag.name != self.ROOT_TAG_NAME: + self.popTag() + + def __getattr__(self, methodName): + """This method routes method call requests to either the SGMLParser + superclass or the Tag superclass, depending on the method name.""" + #print "__getattr__ called on %s.%s" % (self.__class__, methodName) + + if methodName.startswith('start_') or methodName.startswith('end_') \ + or methodName.startswith('do_'): + return SGMLParser.__getattr__(self, methodName) + elif not methodName.startswith('__'): + return Tag.__getattr__(self, methodName) + else: + raise AttributeError + + def isSelfClosingTag(self, name): + """Returns true iff the given string is the name of a + self-closing tag according to this parser.""" + return self.SELF_CLOSING_TAGS.has_key(name) \ + or self.instanceSelfClosingTags.has_key(name) + + def reset(self): + Tag.__init__(self, self, self.ROOT_TAG_NAME) + self.hidden = 1 + SGMLParser.reset(self) + self.currentData = [] + self.currentTag = None + self.tagStack = [] + self.quoteStack = [] + self.pushTag(self) + + def popTag(self): + tag = self.tagStack.pop() + + #print "Pop", tag.name + if self.tagStack: + self.currentTag = self.tagStack[-1] + return self.currentTag + + def pushTag(self, tag): + #print "Push", tag.name + if self.currentTag: + self.currentTag.contents.append(tag) + self.tagStack.append(tag) + self.currentTag = self.tagStack[-1] + + def endData(self, containerClass=NavigableString): + if self.currentData: + currentData = u''.join(self.currentData) + if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and + not set([tag.name for tag in self.tagStack]).intersection( + self.PRESERVE_WHITESPACE_TAGS)): + if '\n' in currentData: + currentData = '\n' + else: + currentData = ' ' + self.currentData = [] + if self.parseOnlyThese and len(self.tagStack) <= 1 and \ + (not self.parseOnlyThese.text or \ + not self.parseOnlyThese.search(currentData)): + return + o = containerClass(currentData) + o.setup(self.currentTag, self.previous) + if self.previous: + self.previous.next = o + self.previous = o + self.currentTag.contents.append(o) + + + def _popToTag(self, name, inclusivePop=True): + """Pops the tag stack up to and including the most recent + instance of the given tag. If inclusivePop is false, pops the tag + stack up to but *not* including the most recent instqance of + the given tag.""" + #print "Popping to %s" % name + if name == self.ROOT_TAG_NAME: + return + + numPops = 0 + mostRecentTag = None + for i in range(len(self.tagStack)-1, 0, -1): + if name == self.tagStack[i].name: + numPops = len(self.tagStack)-i + break + if not inclusivePop: + numPops = numPops - 1 + + for i in range(0, numPops): + mostRecentTag = self.popTag() + return mostRecentTag + + def _smartPop(self, name): + + """We need to pop up to the previous tag of this type, unless + one of this tag's nesting reset triggers comes between this + tag and the previous tag of this type, OR unless this tag is a + generic nesting trigger and another generic nesting trigger + comes between this tag and the previous tag of this type. + + Examples: +

FooBar *

* should pop to 'p', not 'b'. +

FooBar *

* should pop to 'table', not 'p'. +

Foo

Bar *

* should pop to 'tr', not 'p'. + +

    • *
    • * should pop to 'ul', not the first 'li'. +
    • ** should pop to 'table', not the first 'tr' + tag should + implicitly close the previous tag within the same
    ** should pop to 'tr', not the first 'td' + """ + + nestingResetTriggers = self.NESTABLE_TAGS.get(name) + isNestable = nestingResetTriggers != None + isResetNesting = self.RESET_NESTING_TAGS.has_key(name) + popTo = None + inclusive = True + for i in range(len(self.tagStack)-1, 0, -1): + p = self.tagStack[i] + if (not p or p.name == name) and not isNestable: + #Non-nestable tags get popped to the top or to their + #last occurance. + popTo = name + break + if (nestingResetTriggers is not None + and p.name in nestingResetTriggers) \ + or (nestingResetTriggers is None and isResetNesting + and self.RESET_NESTING_TAGS.has_key(p.name)): + + #If we encounter one of the nesting reset triggers + #peculiar to this tag, or we encounter another tag + #that causes nesting to reset, pop up to but not + #including that tag. + popTo = p.name + inclusive = False + break + p = p.parent + if popTo: + self._popToTag(popTo, inclusive) + + def unknown_starttag(self, name, attrs, selfClosing=0): + #print "Start tag %s: %s" % (name, attrs) + if self.quoteStack: + #This is not a real tag. + #print "<%s> is not real!" % name + attrs = ''.join([' %s="%s"' % (x, y) for x, y in attrs]) + self.handle_data('<%s%s>' % (name, attrs)) + return + self.endData() + + if not self.isSelfClosingTag(name) and not selfClosing: + self._smartPop(name) + + if self.parseOnlyThese and len(self.tagStack) <= 1 \ + and (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)): + return + + tag = Tag(self, name, attrs, self.currentTag, self.previous) + if self.previous: + self.previous.next = tag + self.previous = tag + self.pushTag(tag) + if selfClosing or self.isSelfClosingTag(name): + self.popTag() + if name in self.QUOTE_TAGS: + #print "Beginning quote (%s)" % name + self.quoteStack.append(name) + self.literal = 1 + return tag + + def unknown_endtag(self, name): + #print "End tag %s" % name + if self.quoteStack and self.quoteStack[-1] != name: + #This is not a real end tag. + #print " is not real!" % name + self.handle_data('' % name) + return + self.endData() + self._popToTag(name) + if self.quoteStack and self.quoteStack[-1] == name: + self.quoteStack.pop() + self.literal = (len(self.quoteStack) > 0) + + def handle_data(self, data): + self.currentData.append(data) + + def _toStringSubclass(self, text, subclass): + """Adds a certain piece of text to the tree as a NavigableString + subclass.""" + self.endData() + self.handle_data(text) + self.endData(subclass) + + def handle_pi(self, text): + """Handle a processing instruction as a ProcessingInstruction + object, possibly one with a %SOUP-ENCODING% slot into which an + encoding will be plugged later.""" + if text[:3] == "xml": + text = u"xml version='1.0' encoding='%SOUP-ENCODING%'" + self._toStringSubclass(text, ProcessingInstruction) + + def handle_comment(self, text): + "Handle comments as Comment objects." + self._toStringSubclass(text, Comment) + + def handle_charref(self, ref): + "Handle character references as data." + if self.convertEntities: + data = unichr(int(ref)) + else: + data = '&#%s;' % ref + self.handle_data(data) + + def handle_entityref(self, ref): + """Handle entity references as data, possibly converting known + HTML and/or XML entity references to the corresponding Unicode + characters.""" + data = None + if self.convertHTMLEntities: + try: + data = unichr(name2codepoint[ref]) + except KeyError: + pass + + if not data and self.convertXMLEntities: + data = self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref) + + if not data and self.convertHTMLEntities and \ + not self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref): + # TODO: We've got a problem here. We're told this is + # an entity reference, but it's not an XML entity + # reference or an HTML entity reference. Nonetheless, + # the logical thing to do is to pass it through as an + # unrecognized entity reference. + # + # Except: when the input is "&carol;" this function + # will be called with input "carol". When the input is + # "AT&T", this function will be called with input + # "T". We have no way of knowing whether a semicolon + # was present originally, so we don't know whether + # this is an unknown entity or just a misplaced + # ampersand. + # + # The more common case is a misplaced ampersand, so I + # escape the ampersand and omit the trailing semicolon. + data = "&%s" % ref + if not data: + # This case is different from the one above, because we + # haven't already gone through a supposedly comprehensive + # mapping of entities to Unicode characters. We might not + # have gone through any mapping at all. So the chances are + # very high that this is a real entity, and not a + # misplaced ampersand. + data = "&%s;" % ref + self.handle_data(data) + + def handle_decl(self, data): + "Handle DOCTYPEs and the like as Declaration objects." + self._toStringSubclass(data, Declaration) + + def parse_declaration(self, i): + """Treat a bogus SGML declaration as raw data. Treat a CDATA + declaration as a CData object.""" + j = None + if self.rawdata[i:i+9] == '', i) + if k == -1: + k = len(self.rawdata) + data = self.rawdata[i+9:k] + j = k+3 + self._toStringSubclass(data, CData) + else: + try: + j = SGMLParser.parse_declaration(self, i) + except SGMLParseError: + toHandle = self.rawdata[i:] + self.handle_data(toHandle) + j = i + len(toHandle) + return j + +class BeautifulSoup(BeautifulStoneSoup): + + """This parser knows the following facts about HTML: + + * Some tags have no closing tag and should be interpreted as being + closed as soon as they are encountered. + + * The text inside some tags (ie. 'script') may contain tags which + are not really part of the document and which should be parsed + as text, not tags. If you want to parse the text as tags, you can + always fetch it and parse it explicitly. + + * Tag nesting rules: + + Most tags can't be nested at all. For instance, the occurance of + a

    tag should implicitly close the previous

    tag. + +

    Para1

    Para2 + should be transformed into: +

    Para1

    Para2 + + Some tags can be nested arbitrarily. For instance, the occurance + of a

    tag should _not_ implicitly close the previous +
    tag. + + Alice said:
    Bob said:
    Blah + should NOT be transformed into: + Alice said:
    Bob said:
    Blah + + Some tags can be nested, but the nesting is reset by the + interposition of other tags. For instance, a
    , + but not close a tag in another table. + +
    BlahBlah + should be transformed into: +
    BlahBlah + but, + Blah
    Blah + should NOT be transformed into + Blah
    Blah + + Differing assumptions about tag nesting rules are a major source + of problems with the BeautifulSoup class. If BeautifulSoup is not + treating as nestable a tag your page author treats as nestable, + try ICantBelieveItsBeautifulSoup, MinimalSoup, or + BeautifulStoneSoup before writing your own subclass.""" + + def __init__(self, *args, **kwargs): + if not kwargs.has_key('smartQuotesTo'): + kwargs['smartQuotesTo'] = self.HTML_ENTITIES + kwargs['isHTML'] = True + BeautifulStoneSoup.__init__(self, *args, **kwargs) + + SELF_CLOSING_TAGS = buildTagMap(None, + ('br' , 'hr', 'input', 'img', 'meta', + 'spacer', 'link', 'frame', 'base', 'col')) + + PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea']) + + QUOTE_TAGS = {'script' : None, 'textarea' : None} + + #According to the HTML standard, each of these inline tags can + #contain another tag of the same type. Furthermore, it's common + #to actually use these tags this way. + NESTABLE_INLINE_TAGS = ('span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', + 'center') + + #According to the HTML standard, these block tags can contain + #another tag of the same type. Furthermore, it's common + #to actually use these tags this way. + NESTABLE_BLOCK_TAGS = ('blockquote', 'div', 'fieldset', 'ins', 'del') + + #Lists can contain other lists, but there are restrictions. + NESTABLE_LIST_TAGS = { 'ol' : [], + 'ul' : [], + 'li' : ['ul', 'ol'], + 'dl' : [], + 'dd' : ['dl'], + 'dt' : ['dl'] } + + #Tables can contain other tables, but there are restrictions. + NESTABLE_TABLE_TAGS = {'table' : [], + 'tr' : ['table', 'tbody', 'tfoot', 'thead'], + 'td' : ['tr'], + 'th' : ['tr'], + 'thead' : ['table'], + 'tbody' : ['table'], + 'tfoot' : ['table'], + } + + NON_NESTABLE_BLOCK_TAGS = ('address', 'form', 'p', 'pre') + + #If one of these tags is encountered, all tags up to the next tag of + #this type are popped. + RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript', + NON_NESTABLE_BLOCK_TAGS, + NESTABLE_LIST_TAGS, + NESTABLE_TABLE_TAGS) + + NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS, + NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS) + + # Used to detect the charset in a META tag; see start_meta + CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M) + + def start_meta(self, attrs): + """Beautiful Soup can detect a charset included in a META tag, + try to convert the document to that charset, and re-parse the + document from the beginning.""" + httpEquiv = None + contentType = None + contentTypeIndex = None + tagNeedsEncodingSubstitution = False + + for i in range(0, len(attrs)): + key, value = attrs[i] + key = key.lower() + if key == 'http-equiv': + httpEquiv = value + elif key == 'content': + contentType = value + contentTypeIndex = i + + if httpEquiv and contentType: # It's an interesting meta tag. + match = self.CHARSET_RE.search(contentType) + if match: + if (self.declaredHTMLEncoding is not None or + self.originalEncoding == self.fromEncoding): + # An HTML encoding was sniffed while converting + # the document to Unicode, or an HTML encoding was + # sniffed during a previous pass through the + # document, or an encoding was specified + # explicitly and it worked. Rewrite the meta tag. + def rewrite(match): + return match.group(1) + "%SOUP-ENCODING%" + newAttr = self.CHARSET_RE.sub(rewrite, contentType) + attrs[contentTypeIndex] = (attrs[contentTypeIndex][0], + newAttr) + tagNeedsEncodingSubstitution = True + else: + # This is our first pass through the document. + # Go through it again with the encoding information. + newCharset = match.group(3) + if newCharset and newCharset != self.originalEncoding: + self.declaredHTMLEncoding = newCharset + self._feed(self.declaredHTMLEncoding) + raise StopParsing + pass + tag = self.unknown_starttag("meta", attrs) + if tag and tagNeedsEncodingSubstitution: + tag.containsSubstitutions = True + +class StopParsing(Exception): + pass + +class ICantBelieveItsBeautifulSoup(BeautifulSoup): + + """The BeautifulSoup class is oriented towards skipping over + common HTML errors like unclosed tags. However, sometimes it makes + errors of its own. For instance, consider this fragment: + + FooBar + + This is perfectly valid (if bizarre) HTML. However, the + BeautifulSoup class will implicitly close the first b tag when it + encounters the second 'b'. It will think the author wrote + "FooBar", and didn't close the first 'b' tag, because + there's no real-world reason to bold something that's already + bold. When it encounters '' it will close two more 'b' + tags, for a grand total of three tags closed instead of two. This + can throw off the rest of your document structure. The same is + true of a number of other tags, listed below. + + It's much more common for someone to forget to close a 'b' tag + than to actually use nested 'b' tags, and the BeautifulSoup class + handles the common case. This class handles the not-co-common + case: where you can't believe someone wrote what they did, but + it's valid HTML and BeautifulSoup screwed up by assuming it + wouldn't be.""" + + I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \ + ('em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', + 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b', + 'big') + + I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ('noscript',) + + NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS, + I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS, + I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS) + +class MinimalSoup(BeautifulSoup): + """The MinimalSoup class is for parsing HTML that contains + pathologically bad markup. It makes no assumptions about tag + nesting, but it does know which tags are self-closing, that + ")] + data = data[:data.rindex(";")] + data = data.replace('tables:','"tables":') + tables = json.loads(data)['tables'] + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(url) + else: + raise e + + # looks like only one author per story allowed. + author = tables['users'][0] + story = tables['stories'][0] + story_ver = tables['story_versions'][0] + print("story:%s"%story) + + self.story.setMetadata('authorId',author['id']) + self.story.setMetadata('author',author['display_name']) + self.story.setMetadata('authorUrl','https://'+self.host+'/author/'+author['display_name']+'/stories') + + self.story.setMetadata('title',story_ver['title']) + self.setDescription(url,story_ver['description']) + + if not ('assets/story_versions/covers' in story_ver['profile_image_url@2x']): + self.setCoverImage(url,story_ver['profile_image_url@2x']) + + self.story.setMetadata('datePublished',makeDate(story['published_at'], self.dateformat)) + self.story.setMetadata('dateUpdated',makeDate(story['published_at'], self.dateformat)) + + self.story.setMetadata('followers',story['followers_count']) + self.story.setMetadata('comments',story['comments_count']) + self.story.setMetadata('views',story['views_count']) + self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these. + if 'dislikes' in story: + self.story.setMetadata('dislikes',int(story['dislikes'])) + + if story_ver['is_complete']: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + self.story.setMetadata('rating', story_ver['maturity_level']) + self.story.setMetadata('numWords', unicode(story_ver['word_count'])) + + for i in tables['fandoms']: + self.story.addToList('category',i['name']) + + for i in tables['genres']: + self.story.addToList('genre',i['name']) + + for i in tables['characters']: + self.story.addToList('characters',i['name']) + + for c in tables['chapters']: + chtitle = "Chapter %d"%c['number'] + if c['title']: + chtitle += " - %s"%c['title'] + self.chapterUrls.append((chtitle,c['body_url'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + def getChapterText(self, url): + logger.debug('Getting chapter text from: %s' % url) + if not url: + data = u"This chapter has no text." + else: + data = self._fetchUrl(url) + soup = bs.BeautifulSoup(u"
    "+data+u"
    ") + return self.utf8FromSoup(url,soup) + +def getClass(): + return FictionPadSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_fictionpresscom.py b/fanficdownloader/adapters/adapter_fictionpresscom.py new file mode 100644 index 0000000..795ff94 --- /dev/null +++ b/fanficdownloader/adapters/adapter_fictionpresscom.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 +import time + +## They're from the same people and pretty much identical. +from adapter_fanfictionnet import FanFictionNetSiteAdapter + +class FictionPressComSiteAdapter(FanFictionNetSiteAdapter): + + def __init__(self, config, url): + FanFictionNetSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','fpcom') + + @staticmethod + def getSiteDomain(): + return 'www.fictionpress.com' + + @classmethod + def getAcceptDomains(cls): + return ['www.fictionpress.com','m.fictionpress.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/" + + def getSiteURLPattern(self): + return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$" + +def getClass(): + return FictionPressComSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_ficwadcom.py b/fanficdownloader/adapters/adapter_ficwadcom.py new file mode 100644 index 0000000..7934a13 --- /dev/null +++ b/fanficdownloader/adapters/adapter_ficwadcom.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 +import time +import httplib, urllib + +from .. import exceptions as exceptions +from ..htmlcleanup import stripHTML + +from base_adapter import BaseSiteAdapter, makeDate + +class FicwadComSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','fw') + + # get storyId from url--url validation guarantees second part is storyId + self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2]) + + self.username = "NoneGiven" + self.password = "" + + @staticmethod + def getSiteDomain(): + return 'ficwad.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://ficwad.com/story/1234" + + def getSiteURLPattern(self): + return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$" + + def performLogin(self,url): + params = {} + + if self.password: + params['username'] = self.username + params['password'] = self.password + else: + params['username'] = self.getConfig("username") + params['password'] = self.getConfig("password") + + loginUrl = 'http://' + self.getSiteDomain() + '/account/login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['username'])) + d = self._postUrl(loginUrl,params,usecache=False) + + if "Login attempt failed..." in d: + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['username'])) + raise exceptions.FailedToLogin(url,params['username']) + return False + else: + return True + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + def extractChapterUrlsAndMetadata(self): + + # fetch the chapter. From that we will get almost all the + # metadata and chapter list + + url = self.url + logger.debug("URL: "+url) + + # use BeautifulSoup HTML parser to make everything easier to find. + try: + data = self._fetchUrl(url) + # non-existent/removed story urls get thrown to the front page. + if "

    Welcome to FicWad

    " in data: + raise exceptions.StoryDoesNotExist(self.url) + soup = self.make_soup(data) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # if blocked, attempt login. + if soup.find("div",{"class":"blocked"}) or soup.find("li",{"class":"blocked"}): + if self.performLogin(url): # performLogin raises + # FailedToLogin if it fails. + soup = self.make_soup(self._fetchUrl(url,usecache=False)) + + divstory = soup.find('div',id='story') + storya = divstory.find('a',href=re.compile("^/story/\d+$")) + if storya : # if there's a story link in the divstory header, this is a chapter page. + # normalize story URL on chapter list. + self.story.setMetadata('storyId',storya['href'].split('/',)[2]) + url = "http://"+self.getSiteDomain()+storya['href'] + logger.debug("Normalizing to URL: "+url) + self._setURL(url) + try: + soup = self.make_soup(self._fetchUrl(url)) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # if blocked, attempt login. + if soup.find("div",{"class":"blocked"}) or soup.find("li",{"class":"blocked"}): + if self.performLogin(url): # performLogin raises + # FailedToLogin if it fails. + soup = self.make_soup(self._fetchUrl(url,usecache=False)) + + # title - first h4 tag will be title. + titleh4 = soup.find('div',{'class':'storylist'}).find('h4') + self.story.setMetadata('title', stripHTML(titleh4.a)) + + # Find authorid and URL from... author url. + a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+")) + self.story.setMetadata('authorId',a['href'].split('/')[2]) + self.story.setMetadata('authorUrl','http://'+self.host+a['href']) + self.story.setMetadata('author',a.string) + + # description + storydiv = soup.find("div",{"id":"story"}) + self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p) + #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string) + + # most of the meta data is here: + metap = storydiv.find("p",{"class":"meta"}) + self.story.addToList('category',metap.find("a",href=re.compile(r"^/category/\d+")).string) + + # warnings + # [!!] [R] [V] [Y] + spanreq = metap.find("span",{"class":"story-warnings"}) + if spanreq: # can be no warnings. + for a in spanreq.findAll("a"): + self.story.addToList('warnings',a['title']) + + ## perhaps not the most efficient way to parse this, using + ## regexps for each rather than something more complex, but + ## IMO, it's more readable and amenable to change. + metastr = stripHTML(str(metap)).replace('\n',' ').replace('\t',' ').replace(u'\u00a0',' ') + + m = re.match(r".*?Rating: (.+?) -.*?",metastr) + if m: + self.story.setMetadata('rating', m.group(1)) + + m = re.match(r".*?Genres: (.+?) -.*?",metastr) + if m: + for g in m.group(1).split(','): + self.story.addToList('genre',g) + + m = re.match(r".*?Characters: (.*?) -.*?",metastr) + if m: + for g in m.group(1).split(','): + if g: + self.story.addToList('characters',g) + + m = re.match(r".*?Published: ([0-9-]+?) -.*?",metastr) + if m: + self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y-%m-%d")) + + # Updated can have more than one space after it. + m = re.match(r".*?Updated: ([0-9-]+?) +-.*?",metastr) + if m: + self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y-%m-%d")) + + m = re.match(r".*? - ([0-9,]+?) words.*?",metastr) + if m: + self.story.setMetadata('numWords',m.group(1)) + + if metastr.endswith("Complete"): + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + # get the chapter list first this time because that's how we + # detect the need to login. + storylistul = soup.find('ul',{'class':'storylist'}) + if not storylistul: + # no list found, so it's a one-chapter story. + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + chapterlistlis = storylistul.findAll('li') + for chapterli in chapterlistlis: + if "blocked" in chapterli['class']: + # paranoia check. We should already be logged in by now. + raise exceptions.FailedToLogin(url,self.username) + else: + #print "chapterli.h4.a (%s)"%chapterli.h4.a + self.chapterUrls.append((chapterli.h4.a.string, + u'http://%s%s'%(self.getSiteDomain(), + chapterli.h4.a['href']))) + #print "self.chapterUrls:%s"%self.chapterUrls + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + return + + + def getChapterText(self, url): + logger.debug('Getting chapter text from: %s' % url) + soup = self.make_soup(self._fetchUrl(url)) + + span = soup.find('div', {'id' : 'storytext'}) + + if None == span: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,span) + +def getClass(): + return FicwadComSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_fimfictionnet.py b/fanficdownloader/adapters/adapter_fimfictionnet.py new file mode 100644 index 0000000..f390dc7 --- /dev/null +++ b/fanficdownloader/adapters/adapter_fimfictionnet.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 +import cookielib as cl +import json + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return FimFictionNetSiteAdapter + +class FimFictionNetSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','fimficnet') + self.story.setMetadata('storyId', self.parsedUrl.path.split('/',)[2]) + self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/") + self.is_adult = False + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %b %Y" + + @staticmethod + def getSiteDomain(): + return 'www.fimfiction.net' + + @classmethod + def getAcceptDomains(cls): + # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway + return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here" + + def getSiteURLPattern(self): + return r"https?://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*" + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + def doExtractChapterUrlsAndMetadata(self,get_cover=True): + + if self.is_adult or self.getConfig("is_adult"): + cookie = cl.Cookie(version=0, name='view_mature', value='true', + port=None, port_specified=False, + domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False, + path='/story', path_specified=True, + secure=False, + expires=time.time()+10000, + discard=False, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False) + self.cookiejar.set_cookie(cookie) + + ##--------------------------------------------------------------------------------------------------- + ## Get the story's title page. Check if it exists. + + try: + # don't use cache if manual is_adult--should only happen + # if it's an adult story and they don't have is_adult in ini. + data = self.do_fix_blockquotes(self._fetchUrl(self.url, + usecache=(not self.is_adult))) + soup = bs.BeautifulSoup(data) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data: + raise exceptions.StoryDoesNotExist(self.url) + + if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data: + raise exceptions.AdultCheckRequired(self.url) + + if self.password: + params = {} + params['password'] = self.password + data = self._postUrl(self.url, params) + soup = bs.BeautifulSoup(data) + + if not (soup.find('form', {'id' : 'password_form'}) == None): + if self.getConfig('fail_on_password'): + raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url) + else: + raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True) + + ##---------------------------------------------------------------------------------------------------- + ## Extract metadata + + storyContentBox = soup.find('div', {'class':'story_content_box'}) + + # Title + title = storyContentBox.find('a', {'class':re.compile(r'.*\bstory_name\b.*')}) + self.story.setMetadata('title',stripHTML(title)) + + # Author + author = storyContentBox.find('div', {'class':'author'}).find('a') + self.story.setMetadata("author", stripHTML(author)) + #No longer seems to be a way to access Fimfiction's internal author ID + self.story.setMetadata("authorId", self.story.getMetadata("author")) + self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), stripHTML(author))) + + #Rating text is replaced with full words for historical compatibility after the site changed + #on 2014-10-27 + rating = stripHTML(storyContentBox.find('a', {'class':re.compile(r'.*\bcontent-rating-.*')})) + rating = rating.replace("E", "Everyone").replace("T", "Teen").replace("M", "Mature") + self.story.setMetadata("rating", rating) + + # Chapters + for chapter in storyContentBox.findAll('a',{'class':'chapter_link'}): + self.chapterUrls.append((stripHTML(chapter), 'http://'+self.host+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # Status + # In the case of Fimfiction, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled' + # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress' + # and 'Complete' becomes 'Completed'. 'Cancelled' and 'On Hiatus' are passed through, it's easy now for users + # to change/remove if they want with replace_metadata + status = stripHTML(storyContentBox.find('span', {'class':re.compile(r'.*\bcompleted-status-.*')})) + status = status.replace("Incomplete", "In-Progress").replace("Complete", "Completed") + self.story.setMetadata("status", status) + + # Genres and Warnings + # warnings were folded into general categories in the 2014-10-27 site update + categories = storyContentBox.findAll('a', {'class':re.compile(r'.*\bstory_category\b.*')}) + for category in categories: + category = stripHTML(category) + if category == "Gore" or category == "Sex": + self.story.addToList('warnings', category) + else: + self.story.addToList('genre', category) + + # Word count + wordCountText = stripHTML(storyContentBox.find('li', {'class':'bottom'}).find('div', {'class':'word_count'})) + self.story.setMetadata("numWords", re.sub(r'[^0-9]', '', wordCountText)) + + # Cover image + storyImage = storyContentBox.find('div', {'class':'story_image'}) + if storyImage: + coverurl = storyImage.find('a')['href'] + if coverurl.startswith('//'): # fix for img urls missing 'http:' + coverurl = "http:"+coverurl + if get_cover: + self.setCoverImage(self.url,coverurl) + + coverSource = storyImage.find('a', {'class':'source'}) + if coverSource: + self.story.setMetadata('coverSourceUrl', coverSource['href']) + #There's no text associated with the cover source link, so just + #reuse the URL. Makes it clear it's an external link leading + #outside of the fanfic site, at least. + self.story.setMetadata('coverSource', coverSource['href']) + + # fimf has started including extra stuff inside the description div. + descdivstr = u"%s"%storyContentBox.find("div", {"class":"description"}) + hrstr=u"
    " + descdivstr = u'
    '+descdivstr[descdivstr.index(hrstr)+len(hrstr):] + self.setDescription(self.url,descdivstr) + + # Find the newest and oldest chapter dates + storyData = storyContentBox.find('div', {'class':'story_data'}) + oldestChapter = None + newestChapter = None + self.newestChapterNum = None # save for comparing during update. + # Scan all chapters to find the oldest and newest, on + # FiMFiction it's possible for authors to insert new chapters + # out-of-order or change the dates of earlier ones by editing + # them--That WILL break epub update. + for index, chapterDate in enumerate(storyData.findAll('span', {'class':'date'})): + chapterDate = self.ordinal_date_string_to_date(chapterDate.contents[1]) + if oldestChapter == None or chapterDate < oldestChapter: + oldestChapter = chapterDate + if newestChapter == None or chapterDate > newestChapter: + newestChapter = chapterDate + self.newestChapterNum = index + + if newestChapter is None: + #this will only be true when updating metadata for stories that have 0 chapters + #there is a "last modified" date given on the page, extract it and use that. + moddatetag = storyContentBox.find('span', {'class':'last_modified'}) + if not moddatetag is None: + newestChapter = self.ordinal_date_string_to_date(moddatetag('span')[1].text) + + # Date updated + self.story.setMetadata("dateUpdated", newestChapter) + + # Date published + # falls back to oldest chapter date for stories that haven't been officially published yet + pubdatetag = storyContentBox.find('span', {'class':'date_approved'}) + if pubdatetag is None: + if oldestChapter is None: + #this will only be true when updating metadata for stories that have 0 chapters + #and that have never been officially published - a rare occurrence. Fall back to last + #modified date as the publication date, it's all that we've got. + self.story.setMetadata("datePublished", newestChapter) + else: + self.story.setMetadata("datePublished", oldestChapter) + else: + pubDate = self.ordinal_date_string_to_date(pubdatetag('span')[1].text) + self.story.setMetadata("datePublished", pubDate) + + # Characters + chars = storyContentBox.find("div", {"class":"extra_story_data"}) + for character in chars.findAll("a", {"class":"character_icon"}): + self.story.addToList("characters", character['title']) + + # Likes and dislikes + storyToolbar = soup.find('div', {'class':'story-toolbar'}) + likes = storyToolbar.find('span', {'class':'likes'}) + if not likes is None: + self.story.setMetadata("likes", stripHTML(likes)) + dislikes = storyToolbar.find('span', {'class':'dislikes'}) + if not dislikes is None: + self.story.setMetadata("dislikes", stripHTML(dislikes)) + + # Highest view for a chapter and total views + viewSpan = storyToolbar.find('span', {'title':re.compile(r'.*\btotal views\b.*')}) + self.story.setMetadata("views", re.sub(r'[^0-9]', '', stripHTML(viewSpan))) + self.story.setMetadata("total_views", re.sub(r'[^0-9]', '', viewSpan['title'])) + + # Comment count + commentSpan = storyToolbar.find('span', {'title':re.compile(r'.*\bcomments\b.*')}) + self.story.setMetadata("comment_count", re.sub(r'[^0-9]', '', stripHTML(commentSpan))) + + # Short description + descriptionMeta = soup.find('meta', {'property':'og:description'}) + self.story.setMetadata("short_description", stripHTML(descriptionMeta['content'])) + + #groups + if soup.find('button', {'id':'button-view-all-groups'}): + groupResponse = self._fetchUrl("http://www.fimfiction.net/ajax/groups/story_groups_list.php?story=%s" % (self.story.getMetadata("storyId"))) + groupData = json.loads(groupResponse) + groupList = bs.BeautifulSoup(groupData["content"]) + else: + groupList = soup.find('ul', {'id':'story-groups-list'}) + + if not (groupList == None): + for groupName in groupList.findAll('a'): + self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) + self.story.addToList("groups",stripHTML(groupName).replace(',', ';')) + + #sequels + sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels") + if not sequelStoryHeader == None: + sequelContainer = sequelStoryHeader.parent.parent + for sequel in sequelContainer.findAll('a', {'class':'story_link'}): + self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) + self.story.addToList("sequels", stripHTML(sequel).replace(',', ';')) + + #The link to the prequel is embedded in the description text, so erring + #on the side of caution and wrapping this whole thing in a try block. + #If anything goes wrong this probably wasn't a valid prequel link. + try: + description = soup.find('div', {'class':'description'}) + firstHR = description.find("hr") + nextSib = firstHR.nextSibling + if "This story is a sequel to" in nextSib.string: + link = nextSib.nextSibling + if link.name == "a": + self.story.setMetadata("prequelUrl", 'http://'+self.host+link["href"]) + self.story.setMetadata("prequel", stripHTML(link)) + except: + pass + + def ordinal_date_string_to_date(self, datestring): + datestripped=re.sub(r"(\d+)(st|nd|rd|th)", r"\1", datestring.strip()) + return makeDate(datestripped, self.dateformat) + + def hookForUpdates(self,chaptercount): + if self.oldchapters and len(self.oldchapters) > self.newestChapterNum: + print("Existing epub has %s chapters\nNewest chapter is %s. Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1)) + self.oldchapters = self.oldchapters[:self.newestChapterNum] + return len(self.oldchapters) + + def do_fix_blockquotes(self,data): + if self.getConfig('fix_fimf_blockquotes'): + #

    + #

    + # include > in re groups so there's always something in the group. + data = re.sub(r']*>\s*)]*>)',r'\s*)

    ',r'',data) + return data + + def getChapterText(self, url): + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + + soup = bs.BeautifulSoup(data) + if not (soup.find('form', {'id' : 'password_form'}) == None): + if self.password: + params = {} + params['password'] = self.password + data = self._postUrl(url, params) + else: + print("Chapter %s needed password but no password was present" % url) + + data = self.do_fix_blockquotes(data) + + soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'}) + if soup == None: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,soup) diff --git a/fanficdownloader/adapters/adapter_finestoriescom.py b/fanficdownloader/adapters/adapter_finestoriescom.py new file mode 100644 index 0000000..6c99f9d --- /dev/null +++ b/fanficdownloader/adapters/adapter_finestoriescom.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return FineStoriesComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class FineStoriesComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url + self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0]) + if 'storyInfo' in self.story.getMetadata('storyId'): + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','fnst') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y-%m-%d" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'finestories.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/s/1234 http://"+cls.getSiteDomain()+"/s/1234:4010 http://"+cls.getSiteDomain()+"/library/storyInfo.php?id=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Free Registration' in data \ + or "Invalid Password!" in data \ + or "Invalid User Name!" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['theusername'] = self.username + params['thepassword'] = self.password + else: + params['theusername'] = self.getConfig("username") + params['thepassword'] = self.getConfig("password") + params['rememberMe'] = '1' + params['page'] = 'http://'+self.getSiteDomain()+'/' + params['submit'] = 'Login' + + loginUrl = 'http://' + self.getSiteDomain() + '/login.php' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['theusername'])) + + d = self._fetchUrl(loginUrl, params) + + if "My Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['theusername'])) + raise exceptions.FailedToLogin(url,params['theusername']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"/a/\w+")) + self.story.setMetadata('authorId',a['href'].split('/')[2]) + self.story.setMetadata('authorUrl','http://'+self.host+a['href']) + self.story.setMetadata('author',a.text) + + # Find the chapters: + chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$")) + if len(chapters) != 0: + for chapter in chapters: + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href'])) + else: + self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId'))) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # surprisingly, the detailed page does not give enough details, so go to author's page + + skip=0 + i=0 + while i == 0: + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip))) + + a = asoup.findAll('td', {'class' : 'lc2'}) + for lc2 in a: + if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'): + i=1 + break + if a[len(a)-1] == lc2: + skip=skip+10 + + for cat in lc2.findAll('div', {'class' : 'typediv'}): + self.story.addToList('category',cat.text) + + self.story.setMetadata('numWords', lc2.findNext('td', {'class' : 'num'}).text) + + lc4 = lc2.findNext('td', {'class' : 'lc4'}) + + + try: + a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+")) + i = a.parent.text.split('(')[1].split(')')[0] + self.setSeries(a.text, i) + self.story.setMetadata('seriesUrl','http://'+self.host+a['href']) + except: + pass + try: + a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+")) + self.story.addToList("category",a.text) + except: + pass + + for a in lc4.findAll('span', {'class' : 'help'}): + a.extract() + + self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),lc4.text.split('[More Info')[0]) + + for b in lc4.findAll('b'): + label = b.text + value = b.nextSibling + + if 'For Age' in label: + self.story.setMetadata('rating', value) + + if 'Tags' in label: + for genre in value.split(', '): + self.story.addToList('genre',genre) + + if 'Posted' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat)) + + if 'Concluded' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat)) + + status = lc4.find('span', {'class' : 'ab'}) + if status != None: + self.story.setMetadata('status', 'In-Progress') + if "Last Activity" in status.text: + self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat)) + else: + self.story.setMetadata('status', 'Completed') + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + # some big chapters are split over several pages + pager = div.find('span', {'class' : 'pager'}) + if pager != None: + urls=pager.findAll('a') + urls=urls[:len(urls)-1] + + + for ur in urls: + soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div1 = soup.find('div', {'id' : 'story'}) + + # appending next section + last=div.findAll('p') + next=div1.find('span', {'class' : 'conTag'}).nextSibling + + last[len(last)-1]=last[len(last)-1].append(next) + div.append(div1) + + # removing all the left-over stuff + for a in div.findAll('span'): + a.extract() + + for a in div.findAll('h1'): + a.extract() + for a in div.findAll('h2'): + a.extract() + for a in div.findAll('h3'): + a.extract() + for a in div.findAll('h4'): + a.extract() + for a in div.findAll('br'): + a.extract() + for a in div.findAll('div', {'class' : 'date'}): + a.extract() + + a = div.find('form') + if a != None: + b = a.nextSibling + while b != None: + a.extract() + a=b + b=b.nextSibling + + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_grangerenchantedcom.py b/fanficdownloader/adapters/adapter_grangerenchantedcom.py new file mode 100644 index 0000000..831f08f --- /dev/null +++ b/fanficdownloader/adapters/adapter_grangerenchantedcom.py @@ -0,0 +1,311 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return GrangerEnchantedCom + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class GrangerEnchantedCom(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + self.section=self.parsedUrl.path.split('/',)[1] + + # normalized story URL. + if "malfoymanor" in self.parsedUrl.netloc: + self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId')) + self.story.addToList("category","The Manor") + else: + self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','gech') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%b/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'grangerenchanted.com' + + @classmethod + def getAcceptDomains(cls): + return ['grangerenchanted.com','malfoymanor.grangerenchanted.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + if "enchant" in self.section: + loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login' + else: + loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=1" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Rated: NC-17
    etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Read' in label: + self.story.setMetadata('read', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+self.section+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + except: + # I find it hard to care if the series parsing fails + pass + + try: + self.story.setMetadata('reviews', + stripHTML(soup.find('div',{'id':'sort'}). + findAll('a', href=re.compile(r'^reviews.php'))[1])) + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story1'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py new file mode 100644 index 0000000..bddf4b6 --- /dev/null +++ b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','hp') + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.is_adult=False + + # get storyId from url--url validation guarantees query is only psid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId')) + + + @staticmethod + def getSiteDomain(): + return 'www.harrypotterfanfiction.com' + + @classmethod + def getAcceptDomains(cls): + return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+r"(www\.)?"+re.escape("harrypotterfanfiction.com/viewstory.php?psid=")+r"\d+$" + + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def extractChapterUrlsAndMetadata(self): + + url = self.url+'&index=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + ## Title + a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId'))) + self.story.setMetadata('title',stripHTML(a)) + ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995' + if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")): + raise exceptions.AdultCheckRequired(self.url) + + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?showuid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + ## hpcom doesn't give us total words--but it does give + ## us words/chapter. I'd rather add than fetch and + ## parse another page. + words=0 + for tr in soup.find('table',{'class':'text'}).findAll('tr'): + tdstr = tr.findAll('td')[2].string + if tdstr and tdstr.isdigit(): + words+=int(tdstr) + self.story.setMetadata('numWords',str(words)) + + # Find the chapters: + tablelist = soup.find('table',{'class':'text'}) + for chapter in tablelist.findAll('a', href=re.compile(r'\?chapterid=\d+')): + #javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?chapterid=433441&i=1' + # just in case there's tags, like in chapter titles. + chpt=re.sub(r'^.*?(\?chapterid=\d+).*?',r'\1',chapter['href']) + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php'+chpt)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + ## Finding the metadata is a bit of a pain. Desc is the only thing this color. + desctable= soup.find('table',{'bgcolor':'#f0e8e8'}) + self.setDescription(url,desctable) + #self.story.setMetadata('description',stripHTML(desctable)) + + ## Finding the metadata is a bit of a pain. Most of the meta + ## data is in a center.table without a bgcolor. + #for center in soup.findAll('center'): + table = soup.find('table',{'class':'storymaininfo'}) + if table: + metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ') + # Rating: 12+ Story Reviews: 3 + # Chapters: 3 + # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC + # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa + # Status: Completed + # First Published: 2010.09.02 + # Last Published Chapter: 2010.09.28 + # Last Updated: 2010.09.28 + # Favorite Story Of: 1 users + # Warnings: Scenes of a Mild Sexual Nature + + m = re.match(r".*?Status: Completed.*?",metastr) + if m: + self.story.setMetadata('status','Completed') + else: + self.story.setMetadata('status','In-Progress') + + m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr) + if m: + self.story.setMetadata('rating', m.group(1)) + + m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr) + if m: + for g in m.group(1).split(','): + self.story.addToList('genre',g) + + m = re.match(r".*?Characters: (.+?) Genre.*?",metastr) + if m: + for g in m.group(1).split(','): + self.story.addToList('characters',g) + + m = re.match(r".*?Warnings: (.+).*?",metastr) + if m: + for w in m.group(1).split(','): + if w != 'Now Warnings': + self.story.addToList('warnings',w) + + m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr) + if m: + self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d")) + + # Updated can have more than one space after it. + m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr) + if m: + self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d")) + + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + ## most adapters use BeautifulStoneSoup here, but non-Stone + ## allows nested div tags. + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'fluidtext'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) + +def getClass(): + return HarryPotterFanFictionComSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_hennethannunnet.py b/fanficdownloader/adapters/adapter_hennethannunnet.py new file mode 100644 index 0000000..4a5391d --- /dev/null +++ b/fanficdownloader/adapters/adapter_hennethannunnet.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return HennethAnnunNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class HennethAnnunNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','htan') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.henneth-annun.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/stories/chapter.cfm?stid=1234" + + def getSiteURLPattern(self): + return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + + if "We're sorry. This story is not available." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: This story is not available.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('h2', {'id':'page_heading'}) + self.story.setMetadata('title',stripHTML(a)) + + # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" + for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + self.story.setMetadata('numWords', soup.find('tr', {'class':'foot'}).findAll('td')[1].text) + + self.setDescription(url,soup.find('div', {'id':'summary'})) + + # Rated: NC-17
    etc + info = soup.find('div', {'id':'storyinformation'}) + labels=info.findAll('b') + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Completion' in label: + if 'Complete' in value.string: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Rating' in label: + self.story.setMetadata('rating', value.string) + + if 'Era:' in label: + self.story.addToList('category',value.string) + + if 'Genre' in label: + self.story.addToList('genre',value.string) + + labels=info.findAll('strong') + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Author' in label: + value=value.nextSibling + self.story.setMetadata('authorId',value['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+value['href']) + self.story.setMetadata('author',value.string) + + if 'Post' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated:' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + for char in soup.findAll('a', href=re.compile(r"/resources/bios_view.cfm\?scid=\d+")): + self.story.addToList('characters',stripHTML(char)) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'class' : 'block chapter'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_hlfictionnet.py b/fanficdownloader/adapters/adapter_hlfictionnet.py new file mode 100644 index 0000000..96deb95 --- /dev/null +++ b/fanficdownloader/adapters/adapter_hlfictionnet.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return HLFictionNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class HLFictionNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','hlf') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'hlfiction.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title and author + a = soup.find('div', {'id' : 'pagetitle'}) + + aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',aut['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href']) + self.story.setMetadata('author',aut.string) + aut.extract() + + self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)]) + + # Find the chapters: + chapters=soup.find('select') + if chapters != None: + for chapter in chapters.findAll('option'): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])) + else: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}): + a = list.find('a') + if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']: + break + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Rated: NC-17
    etc + labels = list.findAll('span', {'class' : 'classification'}) + for labelspan in labels: + label = labelspan.string + value = labelspan.nextSibling + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'classification': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value[:len(value)-2]) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + for char in value.string.split(', '): + if not 'None' in char: + self.story.addToList('characters',char) + + if 'Genre' in label: + for genre in value.string.split(', '): + if not 'None' in genre: + self.story.addToList('genre',genre) + + if 'Warnings' in label: + for warning in value.string.split(', '): + if not 'None' in warning: + self.story.addToList('warnings',warning) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']: + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_hpfandomnet.py b/fanficdownloader/adapters/adapter_hpfandomnet.py new file mode 100644 index 0000000..16a4cfc --- /dev/null +++ b/fanficdownloader/adapters/adapter_hpfandomnet.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return HPFandomNetAdapterAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /eff part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/eff/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','hpfdm') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y.%m.%d" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.hpfandom.net' # XXX + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/eff/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/eff/'+a['href']) + self.story.setMetadata('author',a.string) + + ## Going to get the rest from the author page. + authdata = self._fetchUrl(self.story.getMetadata('authorUrl')) + # fix a typo in the site HTML so I can find the Characters list. + authdata = authdata.replace('
    ','') + + # hpfandom.net only seems to indicate adult-only by javascript on the story/chapter links. + if "javascript:if (confirm('Slash/het fiction which incorporates sexual situations to a somewhat graphic degree and some violence. ')) location = 'viewstory.php?sid=%s'"%self.story.getMetadata('storyId') in authdata \ + and not (self.is_adult or self.getConfig("is_adult")): + raise exceptions.AdultCheckRequired(self.url) + + authsoup = bs.BeautifulSoup(authdata) + + reviewsa = authsoup.find('a', href="reviews.php?sid="+self.story.getMetadata('storyId')+"&a=") + # + # + # + labels = metablock.findAll('td',{'width':'10%'}) + for td in labels: + label = td.string + value = td.nextSibling.string + #print("\nlabel:%s\nvalue:%s\n"%(label,value)) + + if 'Category' in label and value: + cats = td.parent.findAll('a',href=re.compile(r'categories.php')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label and value: # this site can have Character label with no + # values, apparently. Others as a precaution. + for char in value.split(','): + self.story.addToList('characters',char.strip()) + + if 'Genre' in label and value: + for genre in value.split(','): + self.story.addToList('genre',genre.strip()) + + if 'Warnings' in label and value: + for warning in value.split(','): + if warning.strip() != 'none': + self.story.addToList('warnings',warning.strip()) + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + # There's no good wrapper around the chapter text. :-/ + # There are, however, tables with width=100% just above and below the real text. + data = re.sub(r'

    + metablock = reviewsa.findParent("table") + #print("metablock:%s"%metablock) + + ## Title + titlea = metablock.find('a', href=re.compile("viewstory.php")) + #print("titlea:%s"%titlea) + if titlea == None: + raise exceptions.FailedToDownload("Story URL (%s) not found on author's page, can't use chapter URLs"%url) + self.story.setMetadata('title',stripHTML(titlea)) + + # Find the chapters: !!! hpfandom.net differs from every other + # eFiction site--the sid on viewstory for chapters is + # *different* for each chapter + for chapter in soup.findAll('a', {'href':re.compile(r"viewstory.php\?sid=\d+&i=\d+")}): + m = re.match(r'.*?(viewstory.php\?sid=\d+&i=\d+).*?',chapter['href']) + # just in case there's tags, like in chapter titles. + #print("====chapter===%s"%m.group(1)) + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eff/'+m.group(1))) + + if len(self.chapterUrls) == 0: + self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + summary = metablock.find("td",{"class":"summary"}) + summary.name='span' + self.setDescription(url,summary) + + # words & completed in first row of metablock. + firstrow = stripHTML(metablock.find('tr')) + # A Mother's Love xx Going Grey 1 (G+) by Kiristeen | Reviews - 18 | Words: 27468 | Completed: Yes + m = re.match(r".*?\((?P[^)]+)\).*?Words: (?P\d+).*?Completed: (?PYes|No)",firstrow) + if m != None: + if m.group('rating') != None: + self.story.setMetadata('rating', m.group('rating')) + + if m.group('words') != None: + self.story.setMetadata('numWords', m.group('words')) + + if m.group('status') != None: + if 'Yes' in m.group('status'): + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + + #

    Chapters:4Published:2010.09.29
    Completed:YesUpdated:2010.10.03
    .*?
    ','
    ', + data,count=1,flags=re.DOTALL) + + data = re.sub(r'.*?
    ','
    ', + data,count=1,flags=re.DOTALL) + + soup = bs.BeautifulStoneSoup(data,selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find("div",{'name':'storybody'}) + #print("\n\ndiv:%s\n\n"%div) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_hpfanficarchivecom.py b/fanficdownloader/adapters/adapter_hpfanficarchivecom.py new file mode 100644 index 0000000..b2fa720 --- /dev/null +++ b/fanficdownloader/adapters/adapter_hpfanficarchivecom.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return HPFanficArchiveComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class HPFanficArchiveComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','hpffa') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%B %d, %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'hpfanficarchive.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/stories/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/stories/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Rated: NC-17
    etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + val = labelspan.nextSibling + value = unicode('') + while val and not defaultGetattr(val,'class') == 'label': + value += unicode(val) + val = val.nextSibling + label = labelspan.string + #print("label:%s\nvalue:%s"%(label,value)) + + if 'Summary' in label: + self.setDescription(url,value) + + if 'Rated' in label: + self.story.setMetadata('rating', stripHTML(value)) + + if 'Word count' in label: + self.story.setMetadata('numWords', stripHTML(value)) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Pairing' in label: + ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4')) + for ship in ships: + self.story.addToList('ships',ship.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in stripHTML(value): + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/stories/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_iketernalnet.py b/fanficdownloader/adapters/adapter_iketernalnet.py new file mode 100644 index 0000000..846aee7 --- /dev/null +++ b/fanficdownloader/adapters/adapter_iketernalnet.py @@ -0,0 +1,283 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return IkEternalNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class IkEternalNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ike') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%B %d, %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.ik-eternal.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&warning=1" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + + # Since the warning text can change by warning level, let's + # look for the warning pass url. ksarchive uses + # &warning= -- actually, so do other sites. Must be an + # eFiction book. + + # viewstory.php?sid=1882&warning=4 + # viewstory.php?sid=1654&ageconsent=ok&warning=5 + #print data + #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data,selfClosingTags=('p')) #poor formatting of the paragraphs in the title page + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Rated: NC-17
    etc + asoup = soup.find('div', {'class': 'listbox'}) + for a in asoup.findAll('p'): + a.name='br' + labels = asoup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_imagineeficcom.py b/fanficdownloader/adapters/adapter_imagineeficcom.py new file mode 100644 index 0000000..f32c584 --- /dev/null +++ b/fanficdownloader/adapters/adapter_imagineeficcom.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return ImagineEFicComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class ImagineEFicComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ime') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y.%m.%d" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'imagine.e-fic.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + + # Rated: NC-17
    etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_indeathnet.py b/fanficdownloader/adapters/adapter_indeathnet.py new file mode 100644 index 0000000..5c0175e --- /dev/null +++ b/fanficdownloader/adapters/adapter_indeathnet.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return InDeathNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class InDeathNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + + # normalized story URL. + self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/') + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','idn') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %B %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'indeath.net' + + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/blog/archive/123-story-in-death/" + + def getSiteURLPattern(self): + # http://www.indeath.net/blog/archive/169-ransom-in-death/ + return re.escape("http://")+re.escape(self.getSiteDomain())+r"/blog/(archive/)?(?P\d+)\-(?P[a-z0-9\-]*)/?$" + + + def getDateFromComponents(self, postmonth, postday): + ym = re.search("Entries\ in\ (?PJanuary|February|March|April|May|June|July|August|September|October|November|December)\ (?P\d{4})",postmonth) + d = re.search("(?P\d{2})\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)",postday) + postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat) + return postdate + + def getAuthorData(self): + + mainUrl = self.url.replace("/archive","") + + try: + maindata = self._fetchUrl(mainUrl) + + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.meta) + else: + raise e + + # use BeautifulSoup HTML parser to make everything easier to find. + mainsoup = bs.BeautifulSoup(maindata) + + # find first entry + e = mainsoup.find('div',{'class':"entry"}) + + # get post author as author + d = e.find('div',{'class':"desc"}) + a = d.find('strong') + self.story.setMetadata('author',a.contents[0].string.strip()) + + # Don't seem to be able to get author pages anymore + self.story.setMetadata('authorUrl','http://www.indeath.net/') + self.story.setMetadata('authorId','0') + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + try: + data = self._fetchUrl(url) + + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.meta) + else: + raise e + + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + # Now go hunting for all the meta data and the chapter list. + + ## Title + h = soup.find('a', id="blog_title") + t = h.find('span') + self.story.setMetadata('title',stripHTML(t.contents[0]).strip()) + + s = t.find('div') + if s != None: + self.setDescription(url,s) + + # Get Author from main blog page since it's not reliably on the archive page + self.getAuthorData() + + # Find the chapters: + chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$")) + + #reverse the list since newest at the top + chapters.reverse() + + # Get date published & updated from first & last entries + posttable=soup.find('div', id="main_column") + + postmonths=posttable.findAll('th', text=re.compile(r'Entries\ in\ ')) + postmonths.reverse() + + postdates=posttable.findAll('span', _class="desc", text=re.compile('\d{2}\ (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)')) + postdates.reverse() + + self.story.setMetadata('datePublished',self.getDateFromComponents(postmonths[0],postdates[0])) + self.story.setMetadata('dateUpdated',self.getDateFromComponents(postmonths[len(postmonths)-1],postdates[len(postdates)-1])) + + # Process List of Chapters + self.story.setMetadata('numChapters',len(chapters)) + logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters')) + for x in range(0,len(chapters)): + # just in case there's tags, like in chapter titles. + chapter=chapters[x] + if len(chapters)==1: + self.chapterUrls.append((self.story.getMetadata('title'),chapter['href'])) + else: + ct = stripHTML(chapter) + tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P.*)$",ct) + if tnew: + chaptertitle = tnew.group('newtitle') + else: + chaptertitle = ct + self.chapterUrls.append((chaptertitle,chapter['href'])) + + + + # grab the text for an individual chapter. + def getChapterText(self, url): + logger.debug('Getting chapter text from: %s' % url) + + #chapter=bs.BeautifulSoup('
    ') + data = self._fetchUrl(url) + soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center')) + + chapter = soup.find("div", "entry_content") + + if None == chapter: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,chapter) + diff --git a/fanficdownloader/adapters/adapter_ksarchivecom.py b/fanficdownloader/adapters/adapter_ksarchivecom.py new file mode 100644 index 0000000..85e47de --- /dev/null +++ b/fanficdownloader/adapters/adapter_ksarchivecom.py @@ -0,0 +1,329 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# Search for XXX comments--that's where things are most likely to need changing. + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return KSArchiveComAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class KSArchiveComAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ksa') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%b/%d/%Y" # XXX + + @classmethod + def getAcceptDomains(cls): + return ['www.ksarchive.com','ksarchive.com'] + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'ksarchive.com' # XXX + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + + # Furthermore, there's a couple sites now with more than + # one warning level for different ratings. And they're + # fussy about it. midnightwhispers has three: 10, 3 & 5. + # we'll try 5 first. + addurl = "&ageconsent=ok&warning=2" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + + # Since the warning text can change by warning level, let's + # look for the warning pass url. ksarchive uses + # &warning= -- actually, so do other sites. Must be an + # eFiction book. + + # viewstory.php?sid=1882&warning=4 + # viewstory.php?sid=1654&ageconsent=ok&warning=5 + #print data + #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) # title's inside a tag. + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',stripHTML(a)) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Rated: NC-17
    etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = stripHTML(labelspan) + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + # poor HTML(unclosed

    for one) can cause run on + # over the next label. + if '' in svalue: + svalue = svalue[0:svalue.find('')] + break + else: + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [stripHTML(cat) for cat in cats] + for cat in catstext: + # ran across one story with an empty + # tag in the desc once. + if cat and cat.strip() in ('Poetry','Essays'): + self.story.addToList('category',stripHTML(cat)) + + if 'Characters' in label: + self.story.addToList('characters','Kirk') + self.story.addToList('characters','Spock') + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [stripHTML(char) for char in chars] + for char in charstext: + self.story.addToList('characters',stripHTML(char)) + + ## Not all sites use Genre, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX + genrestext = [stripHTML(genre) for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',stripHTML(genre)) + + ## In addition to Genre (which is very site specific) KSA + ## has 'Story Type', which is much more what most sites + ## call genre. + if 'Story Type' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX + genrestext = [stripHTML(genre) for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',stripHTML(genre)) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [stripHTML(warning) for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',stripHTML(warning)) + + if 'Universe' in label: + universes = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX + universestext = [stripHTML(universe) for universe in universes] + self.universe = ', '.join(universestext) + for universe in universestext: + self.story.addToList('universe',stripHTML(universe)) + + if 'Crossover Fandom' in label: + crossoverfandoms = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4')) # XXX + crossoverfandomstext = [stripHTML(crossoverfandom) for crossoverfandom in crossoverfandoms] + self.crossoverfandom = ', '.join(crossoverfandomstext) + for crossoverfandom in crossoverfandomstext: + self.story.addToList('crossoverfandom',stripHTML(crossoverfandom)) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = stripHTML(a) + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + soup = bs.BeautifulStoneSoup(data, + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + if "A fatal MySQL error was encountered" in data: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Database error on the site reported!" % url) + else: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_libraryofmoriacom.py b/fanficdownloader/adapters/adapter_libraryofmoriacom.py new file mode 100644 index 0000000..3df41bd --- /dev/null +++ b/fanficdownloader/adapters/adapter_libraryofmoriacom.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + + +def getClass(): + return LibraryOfMoriaComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class LibraryOfMoriaComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','lom') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%B %d, %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.libraryofmoria.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/a/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/a/viewstory.php?sid=")+r"\d+$" + + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + addurl = "&ageconsent=ok&warning=3" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/a/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/a/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Rated: NC-17
    etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + if 'Type' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warning' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/a/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_literotica.py b/fanficdownloader/adapters/adapter_literotica.py new file mode 100644 index 0000000..1a757b6 --- /dev/null +++ b/fanficdownloader/adapters/adapter_literotica.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 +import urlparse +import time + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class LiteroticaSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["utf8", + "Windows-1252"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + + self.story.setMetadata('siteabbrev','litero') + + # normalize to first chapter. Not sure if they ever have more than 2 digits. + storyId = self.parsedUrl.path.split('/',)[2] + # replace later chapters with first chapter but don't remove numbers + # from the URL that disambiguate stories with the same title. + storyId = re.sub("-ch-?\d\d", "", storyId) + self.story.setMetadata('storyId', storyId) + + ## accept m(mobile)url, but use www. + url = re.sub("^(www|german|spanish|french|dutch|italian|romanian|portuguese|other)\.i", + "\1", + url) + + ## strip ?page=... + url = re.sub("\?page=.*$", "", url) + + ## set url + self._setURL(url) + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = '%m/%d/%y' + + @staticmethod + def getSiteDomain(): + return 'literotica.com' + + @classmethod + def getAcceptDomains(cls): + return ['www.literotica.com', + 'www.i.literotica.com', + 'german.literotica.com', + 'german.i.literotica.com', + 'spanish.literotica.com', + 'spanish.i.literotica.com', + 'french.literotica.com', + 'french.i.literotica.com', + 'dutch.literotica.com', + 'dutch.i.literotica.com', + 'italian.literotica.com', + 'italian.i.literotica.com', + 'romanian.literotica.com', + 'romanian.i.literotica.com', + 'portuguese.literotica.com', + 'portuguese.i.literotica.com', + 'other.literotica.com', + 'other.i.literotica.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title" + + def getSiteURLPattern(self): + return r"https?://(www|german|spanish|french|dutch|italian|romanian|portuguese|other)(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)" + + def extractChapterUrlsAndMetadata(self): + """ + NOTE: Some stories can have versions, + e.g. /my-story-ch-05-version-10 + NOTE: If two stories share the same title, a running index is added, + e.g.: /my-story-ch-02-1 + Strategy: + * Go to author's page, search for the current story link, + * If it's in a tr.root-story => One-part story + * , get metadata and be done + * If it's in a tr.sl => Chapter in series + * Search up from there until we find a tr.ser-ttl (this is the + story) + * Gather metadata + * Search down from there for all tr.sl until the next + tr.ser-ttl, foreach + * Chapter link is there + """ + + if not (self.is_adult or self.getConfig("is_adult")): + raise exceptions.AdultCheckRequired(self.url) + + logger.debug("Chapter/Story URL: <%s> " % self.url) + try: + data1 = self._fetchUrl(self.url) + soup1 = bs.BeautifulSoup(data1) + #strip comments from soup + [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))] + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # author + a = soup1.find("span", "b-story-user-y") + self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'][0]) + authorurl = a.a['href'] + if authorurl.startswith('//'): + authorurl = self.parsedUrl.scheme+':'+authorurl + self.story.setMetadata('authorUrl', authorurl) + self.story.setMetadata('author', a.text) + + # get the author page + try: + dataAuth = self._fetchUrl(authorurl) + soupAuth = bs.BeautifulSoup(dataAuth) + #strip comments from soup + [comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, bs.Comment))] + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(authorurl) + else: + raise e + + ## Find link to url in author's page + ## site has started using //domain.name/asdf urls remove https?: from front + storyLink = soupAuth.find('a', href=self.url[self.url.index(':')+1:]) + + if storyLink is not None: + urlTr = storyLink.parent.parent + if urlTr['class'] == "sl": + isSingleStory = False + else: + isSingleStory = True + else: + raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (url, authorurl)) + + if isSingleStory: + self.story.setMetadata('title', storyLink.text) + self.story.setMetadata('description', urlTr.findAll("td")[1].text) + self.story.addToList('eroticatags', urlTr.findAll("td")[2].text) + date = urlTr.findAll('td')[-1].text + self.story.setMetadata('datePublished', makeDate(date, self.dateformat)) + self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat)) + self.chapterUrls = [(storyLink.text, self.url)] + else: + seriesTr = urlTr.previousSibling + while seriesTr['class'] != 'ser-ttl': + seriesTr = seriesTr.previousSibling + m = re.match("^(?P.*?):\s(?P<numChapters>\d+)\sPart\sSeries$", seriesTr.find("strong").text) + self.story.setMetadata('title', m.group('title')) + + ## Walk the chapters + chapterTr = seriesTr.nextSibling + self.chapterUrls = [] + dates = [] + descriptions = [] + while chapterTr is not None and chapterTr['class'] == 'sl': + descriptions.append(chapterTr.findAll("td")[1].text) + chapterLink = chapterTr.find("td", "fc").find("a") + self.chapterUrls.append((chapterLink.text, "http:" + chapterLink["href"])) + self.story.addToList('eroticatags', chapterTr.findAll("td")[2].text) + dates.append(makeDate(chapterTr.findAll('td')[-1].text, self.dateformat)) + chapterTr = chapterTr.nextSibling + + ## Set description to joint chapter descriptions + self.story.setMetadata('description', " / ".join(descriptions)) + + ## Set the oldest date as publication date, the newest as update date + dates.sort() + self.story.setMetadata('datePublished', dates[0]) + self.story.setMetadata('dateUpdated', dates[-1]) + + # normalize on first chapter URL. + self._setURL(self.chapterUrls[0][1]) + + self.story.setMetadata('numChapters', len(self.chapterUrls)) + + # set storyId to 'title-author' to avoid duplicates + # self.story.setMetadata('storyId', + # re.sub("[^a-z0-9]", "", self.story.getMetadata('title').lower()) + # + "-" + # + re.sub("[^a-z0-9]", "", self.story.getMetadata('author').lower())) + + return + + def getChapterText(self, url): + logger.debug('Getting chapter text from <%s>' % url) + data1 = self._fetchUrl(url) + # brute force approach to replace the wrapping <p> tag. If + # done by changing tag name, it causes problems with nested + # <p> tags. + data1 = data1.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>') + soup1 = bs.BeautifulSoup(data1) + + #strip comments from soup + [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))] + + # get story text + story1 = soup1.find('div', 'b-story-body-x').div + #print("story1:%s"%story1) + # story1.name='div' + story1.append('<br />') + storytext = self.utf8FromSoup(url,story1) + + # find num pages + pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0]) + logger.debug("pages: "+str(pgs)) + + # get all the pages + for i in xrange(2, pgs+1): + try: + logger.debug("fetching page "+str(i)) + time.sleep(0.5) + data2 = self._fetchUrl(url, {'page': i}) + # brute force approach to replace the wrapping <p> tag. If + # done by changing tag name, it causes problems with nested + # <p> tags. + data2 = data2.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>') + soup2 = bs.BeautifulSoup(data2) + [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))] + story2 = soup2.find('div', 'b-story-body-x').div + # story2.name='div' + story2.append('<br />') + storytext += self.utf8FromSoup(url,story2) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(url) + else: + raise e + return storytext + + +def getClass(): + return LiteroticaSiteAdapter + + + diff --git a/fanficdownloader/adapters/adapter_lotrfanfictioncom.py b/fanficdownloader/adapters/adapter_lotrfanfictioncom.py new file mode 100644 index 0000000..b20c2ec --- /dev/null +++ b/fanficdownloader/adapters/adapter_lotrfanfictioncom.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +from base_efiction_adapter import BaseEfictionAdapter + +class TheLOTRFanFictionSiteAdapter(BaseEfictionAdapter): + + @staticmethod + def getSiteDomain(): + return 'lotrfanfiction.com' + + @classmethod + def getSiteAbbrev(seluuf): + return 'lotrff' + + @classmethod + def getDateFormat(self): + return "%d/%m/%y" + +def getClass(): + return TheLOTRFanFictionSiteAdapter diff --git a/fanficdownloader/adapters/adapter_lumossycophanthexcom.py b/fanficdownloader/adapters/adapter_lumossycophanthexcom.py new file mode 100644 index 0000000..805d752 --- /dev/null +++ b/fanficdownloader/adapters/adapter_lumossycophanthexcom.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return LumosSycophantHexComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class LumosSycophantHexComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','lsph') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'lumos.sycophanthex.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=19" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if "Age Consent Required" in data: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + pt = soup.find('div', {'id' : 'pagetitle'}) + a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + rating=pt.text.split('(')[1].split(')')[0] + self.story.setMetadata('rating', rating) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + + # <span class="label">Rated:</span> NC-17<br /> etc + + labels = soup.findAll('span',{'class':'label'}) + + value = labels[0].previousSibling + svalue = "" + while value != None: + val = value + value = value.previousSibling + while not defaultGetattr(val,'class') == 'label': + svalue += str(val) + val = val.nextSibling + self.setDescription(url,svalue) + + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Word count' in label: + self.story.setMetadata('numWords', value.split(' -')[0]) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Complete' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_mediaminerorg.py b/fanficdownloader/adapters/adapter_mediaminerorg.py new file mode 100644 index 0000000..261b1f7 --- /dev/null +++ b/fanficdownloader/adapters/adapter_mediaminerorg.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class MediaMinerOrgSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','mm') + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId')) + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + @staticmethod + def getSiteDomain(): + return 'www.mediaminer.org' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+cls.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c" + + def getSiteURLPattern(self): + ## http://www.mediaminer.org/fanfic/view_st.php/76882 + ## http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c + return re.escape("http://"+self.getSiteDomain())+\ + "/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$" + + def extractChapterUrlsAndMetadata(self): + + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + # [ A - All Readers ], strip '[' ']' + ## Above title because we remove the smtxt font to get title. + smtxt = soup.find("font",{"class":"smtxt"}) + if not smtxt: + raise exceptions.StoryDoesNotExist(self.url) + rating = smtxt.string[1:-1] + self.story.setMetadata('rating',rating) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"/fanfic/src.php/u/\d+")) + self.story.setMetadata('authorId',a['href'].split('/')[-1]) + self.story.setMetadata('authorUrl','http://'+self.host+a['href']) + self.story.setMetadata('author',a.string) + + ## Title - Good grief. Title varies by chaptered, 1chapter and 'type=one shot'--and even 'one-shot's can have titled chapter. + ## But, if colspan=2, there's no chapter title. + ## <td class="ffh">Atmosphere: Chapter 1</b> <font class="smtxt">[ P - Pre-Teen ]</font></td> + ## <td colspan=2 class="ffh">Hearts of Ice <font class="smtxt">[ P - Pre-Teen ]</font></td> + ## <td colspan=2 class="ffh">Suzaku no Princess <font class="smtxt">[ P - Pre-Teen ]</font></td> + ## <td class="ffh">The Kraut, The Bartender, and The Drunkard: Chapter 1</b> <font class="smtxt">[ P - Pre-Teen ]</font></td> + ## <td class="ffh">Betrayal and Justice: A Cold Heart</b> <font size="-1">( Chapter 1 )</font> <font class="smtxt">[ A - All Readers ]</font></td> + ## <td class="ffh">Question and Answer: Question and Answer</b> <font size="-1">( One-Shot )</font> <font class="smtxt">[ A - All Readers ]</font></td> + title = soup.find('td',{'class':'ffh'}) + for font in title.findAll('font'): + font.extract() # removes 'font' tags from inside the td. + if title.has_key('colspan'): + titlet = stripHTML(title) + else: + ## No colspan, it's part chapter title--even if it's a one-shot. + titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title + self.story.setMetadata('title',titlet) + ## The story title is difficult to reliably parse from the + ## story pages. Getting it from the author page is, but costs + ## another fetch. + # authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + # titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')}) + # self.story.setMetadata('title',titlea.text) + + # save date from first for later. + firstdate=None + + # Find the chapters + select = soup.find('select',{'name':'cid'}) + if not select: + self.chapterUrls.append(( self.story.getMetadata('title'),self.url)) + else: + for option in select.findAll("option"): + chapter = stripHTML(option.string) + ## chapter can be: Chapter 7 [Jan 23, 2011] + ## or: Vigilant Moonlight ( Chapter 1 ) [Jan 30, 2004] + ## or even: Prologue ( Prologue ) [Jul 31, 2010] + m = re.match(r'^(.*?) (\( .*? \) )?\[(.*?)\]$',chapter) + chapter = m.group(1) + # save date from first for later. + if not firstdate: + firstdate = m.group(3) + self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value'])) + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # category + # <a href="/fanfic/src.php/a/567">Ranma 1/2</a> + for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/a/")): + self.story.addToList('category',a.string) + + # genre + # <a href="/fanfic/src.php/a/567">Ranma 1/2</a> + for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/g/")): + self.story.addToList('genre',a.string) + + # if firstdate, then the block below will only have last updated. + if firstdate: + self.story.setMetadata('datePublished', makeDate(firstdate, "%b %d, %Y")) + # Everything else is in <tr bgcolor="#EEEED4"> + + metastr = stripHTML(soup.find("tr",{"bgcolor":"#EEEED4"})).replace('\n',' ').replace('\r',' ').replace('\t',' ') + # Latest Revision: August 03, 2010 + m = re.match(r".*?(?:Latest Revision|Uploaded On): ([a-zA-Z]+ \d\d, \d\d\d\d)",metastr) + if m: + self.story.setMetadata('dateUpdated', makeDate(m.group(1), "%B %d, %Y")) + if not firstdate: + self.story.setMetadata('datePublished', + self.story.getMetadataRaw('dateUpdated')) + + else: + self.story.setMetadata('dateUpdated', + self.story.getMetadataRaw('datePublished')) + + # Words: 123456 + m = re.match(r".*?\| Words: (\d+) \|",metastr) + if m: + self.story.setMetadata('numWords', m.group(1)) + + # Summary: .... + m = re.match(r".*?Summary: (.*)$",metastr) + if m: + self.setDescription(url, m.group(1)) + #self.story.setMetadata('description', m.group(1)) + + # completed + m = re.match(r".*?Status: Completed.*?",metastr) + if m: + self.story.setMetadata('status','Completed') + else: + self.story.setMetadata('status','In-Progress') + + return + + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data=self._fetchUrl(url) + soup = bs.BeautifulStoneSoup(data, + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + anchor = soup.find('a',{'name':'fic_c'}) + + if None == anchor: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + ## find divs with align=left, those are paragraphs in newer stories. + divlist = anchor.findAllNext('div',{'align':'left'}) + if divlist: + for div in divlist: + div.name='p' # convert to <p> mediaminer uses div with + # a margin for paragraphs. + anchor.append(div) # cheat! stuff all the content + # divs into anchor just as a + # holder. + del div['style'] + del div['align'] + anchor.name='div' + return self.utf8FromSoup(url,anchor) + + else: + logger.debug('Using kludgey text find for older mediaminer story.') + ## Some older mediaminer stories are unparsable with BeautifulSoup. + ## Really nasty formatting. Sooo... Cheat! Parse it ourselves a bit first. + ## Story stuff falls between: + data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>" + soup = bs.BeautifulStoneSoup(data, + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + for tag in soup.findAll('td',{'class':'ffh'}) + \ + soup.findAll('div',{'class':'acl'}) + \ + soup.findAll('div',{'class':'footer smtxt'}) + \ + soup.findAll('table',{'class':'tbbrdr'}): + tag.extract() # remove tag from soup. + + return self.utf8FromSoup(url,soup) + + +def getClass(): + return MediaMinerOrgSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py b/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py new file mode 100644 index 0000000..ea518ea --- /dev/null +++ b/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py @@ -0,0 +1,294 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return MerlinFicDtwinsCoUk + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class MerlinFicDtwinsCoUk(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','mrfd') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%b %d, %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'merlinfic.dtwins.co.uk' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Pairing' in label: + ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for ship in ships: + self.story.addToList('ships',ship.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_midnightwhispersca.py b/fanficdownloader/adapters/adapter_midnightwhispersca.py new file mode 100644 index 0000000..ec884d6 --- /dev/null +++ b/fanficdownloader/adapters/adapter_midnightwhispersca.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# Search for XXX comments--that's where things are most likely to need changing. + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return MidnightwhispersCaAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','mw') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%B %d, %Y" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.midnightwhispers.ca' # XXX + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + + # Furthermore, there's a couple sites now with more than + # one warning level for different ratings. And they're + # fussy about it. midnightwhispers has three: 10, 3 & 5. + # we'll try 5 first. + addurl = "&ageconsent=ok&warning=5" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + + # Since the warning text can change by warning level, let's + # look for the warning pass url. nfacommunity uses + # &warning= -- actually, so do other sites. Must be an + # eFiction book. + + # viewstory.php?sid=1882&warning=4 + # viewstory.php?sid=1654&ageconsent=ok&warning=5 + #print data + #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag. + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + ## Not all sites use Genre, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + soup = bs.BeautifulStoneSoup(data, + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + if "A fatal MySQL error was encountered" in data: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Database error on the site reported!" % url) + else: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_mugglenetcom.py b/fanficdownloader/adapters/adapter_mugglenetcom.py new file mode 100644 index 0000000..261a623 --- /dev/null +++ b/fanficdownloader/adapters/adapter_mugglenetcom.py @@ -0,0 +1,336 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return MuggleNetComAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class MuggleNetComAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','mgln') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. + return 'fanfiction.mugglenet.com' + + @classmethod + def getAcceptDomains(cls): + return ['fanfiction.mugglenet.com','fanfic.mugglenet.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if "class='errortext'>Registered Users Only" in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId') + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + # http://fanfiction.mugglenet.com/viewstory.php?sid=91079&ageconsent=ok&warning=3 + addurl = "&ageconsent=ok&warning=3" # XXX &warning=5 + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + #print("\nurl:%s\ndata:\n%s\n"%(url,data)) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + + # Since the warning text can change by warning level, let's + # look for the warning pass url. nfacommunity uses + # &warning= -- actually, so do other sites. Must be an + # eFiction book. + + # viewstory.php?sid=1882&warning=4 + # viewstory.php?sid=1654&ageconsent=ok&warning=5 + #print data + #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) + m = re.search(r"'viewstory.php\?sid=%s((?:&ageconsent=ok)?&warning=\d+)'"%self.story.getMetadata('storyId'),data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + + # Not good enough-- content can contain a ('), which ends the content prematurely. + # metadesc = soup.find('meta',{'name':'description'}) + # print("removeAllEntities(metadesc['content']):\n%s\n"%removeAllEntities(metadesc['content'])) + start='<span class="label">Summary: </span>' + end='<span class="label">Rated:</span>' + summarydata = data[data.index(start)+len(start):data.index(end)] + #print("summarydata:\n%s\n"%summarydata) + self.setDescription(url,bs.BeautifulSoup(summarydata)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + # not good enough--poorly formated summary html will break it. + # if 'Summary' in label: + # ## Everything until the next span class='label' + # svalue = "" + # while not defaultGetattr(value,'class') == 'label': + # svalue += str(value) + # value = value.nextSibling + # self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + ## Not all sites use Genre, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_nationallibrarynet.py b/fanficdownloader/adapters/adapter_nationallibrarynet.py new file mode 100644 index 0000000..c9d7531 --- /dev/null +++ b/fanficdownloader/adapters/adapter_nationallibrarynet.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return NationalLibraryNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class NationalLibraryNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only storyid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ntlb') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m-%d-%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + return 'national-library.net' + + @classmethod + def getAcceptDomains(cls): + return ['www.national-library.net','national-library.net'] + + @classmethod + def getSiteExampleURLs(cls): + # ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: + return "http://"+cls.getSiteDomain()+"/viewstory.php?storyid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$" + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('h1') + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for p in soup.findAll('p'): + chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$")) + if len(chapters) > 0: + for chapter in chapters: + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + break + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + self.story.setMetadata('status', 'Completed') + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('b') + for x in range(2,len(labels)): + value = labels[x].nextSibling + label = labels[x].string + + if 'Summary' in label: + self.setDescription(url,value) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rating' in label: + self.story.setMetadata('rating', stripHTML(value.nextSibling)) + + if 'Word Count' in label: + self.story.setMetadata('numWords', value.string) + + if 'Category' in label: + for cat in value.string.split(', '): + self.story.addToList('category',cat) + if 'Crossover Shows' in label: + for cat in value.string.split(', '): + if "No Show" not in cat: + self.story.addToList('category',cat) + + if 'Character' in label: + for char in value.string.split(', '): + self.story.addToList('characters',char) + + if 'Pairing' in label: + for char in value.string.split(', '): + self.story.addToList('ships',char) + + if 'Warnings' in label: + for warning in value.string.split(', '): + self.story.addToList('warnings',warning) + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Series' in label: + self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:]) + self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href']) + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId'))) + + a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ') + for genre in a: + self.story.setMetadata('genre', genre) + + a=story.findNext(text=re.compile('Archived')) + self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat)) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div') + + # bit messy since higly inconsistent + for p in soup.findAll('p', {'align' : 'center'}): + p.extract() + p = soup.findAll('p') + for x in range(0,3): + p[x].extract() + if "Chapters: " in stripHTML(p[3]): + p[3].extract() + for x in range(len(p)-2,len(p)-1): + p[x].extract() + + for p in soup.findAll('h1'): + p.extract() + for p in soup.findAll('h3'): + p.extract() + for p in soup.findAll('a'): + p.extract() + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_ncisficcom.py b/fanficdownloader/adapters/adapter_ncisficcom.py new file mode 100644 index 0000000..dbbf3fb --- /dev/null +++ b/fanficdownloader/adapters/adapter_ncisficcom.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return NCISFicComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class NCISFicComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only storyid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ncisf') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m-%d-%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + return 'ncisfic.com' + + @classmethod + def getAcceptDomains(cls): + return ['www.ncisfic.com','ncisfic.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?storyid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$" + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('h1') + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for p in soup.findAll('p'): + chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$")) + if len(chapters) > 0: + for chapter in chapters: + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + break + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + self.story.setMetadata('status', 'Completed') + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('b') + for x in range(2,len(labels)): + value = labels[x].nextSibling + label = labels[x].string + + if 'Summary' in label: + self.setDescription(url,value) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rating' in label: + self.story.setMetadata('rating', stripHTML(value.nextSibling)) + + if 'Word Count' in label: + self.story.setMetadata('numWords', value.string) + + if 'Category' in label: + for cat in value.string.split(', '): + self.story.addToList('category',cat) + if 'Crossover Shows' in label: + for cat in value.string.split(', '): + if "No Show" not in cat: + self.story.addToList('category',cat) + + if 'Character' in label: + for char in value.string.split(', '): + self.story.addToList('characters',char) + + if 'Pairing' in label: + for char in value.string.split(', '): + self.story.addToList('ships',char) + + if 'Warnings' in label: + for warning in value.string.split(', '): + self.story.addToList('warnings',warning) + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Series' in label: + if "No Series" not in value.nextSibling.string: + self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:]) + self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href']) + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId'))) + + a=story.findNext('font') + if 'Complete' in a.string: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ') + for genre in a: + self.story.setMetadata('genre', genre) + + a=story.findNext(text=re.compile('Archived')) + self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat)) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div') + + # bit messy since higly inconsistent + for p in soup.findAll('p', {'align' : 'center'}): + p.extract() + p = soup.findAll('p') + for x in range(0,3): + p[x].extract() + if "Chapters: " in stripHTML(p[3]): + p[3].extract() + for x in range(len(p)-2,len(p)-1): + p[x].extract() + + for p in soup.findAll('h1'): + p.extract() + for p in soup.findAll('h3'): + p.extract() + for p in soup.findAll('a'): + p.extract() + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_ncisfictionnet.py b/fanficdownloader/adapters/adapter_ncisfictionnet.py new file mode 100644 index 0000000..58abedf --- /dev/null +++ b/fanficdownloader/adapters/adapter_ncisfictionnet.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return NCISFictionNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class NCISFictionNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["iso-8859-1", + "Windows-1252"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL("http://"+self.getSiteDomain()\ + +"/chapters.php?stid="+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ncisfn') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%m/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.ncisfiction.net' + + ## Changed from www.ncisfiction.com to www.ncisfiction.net Oct + ## 2012 due to the ncisfiction.com domain expiring. Still accept + ## .com domains for existing updates, etc. + + @classmethod + def getAcceptDomains(cls): + return ['www.ncisfiction.net','www.ncisfiction.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/story.php?stid=01234 http://"+cls.getSiteDomain()+"/chapters.php?stid=1234" + + def getSiteURLPattern(self): + return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+' + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulStoneSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title and author + a = soup.find('div', {'class' : 'main_title'}) + + aut = a.find('a') + self.story.setMetadata('authorId',aut['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href']) + self.story.setMetadata('author',aut.string) + + aut.extract() + self.story.setMetadata('title',stripHTML(a)[:len(stripHTML(a))-2]) + + # Find the chapters: + i=0 + chapters=soup.findAll('table', {'class' : 'story_table'}) + for chapter in chapters: + ch=chapter.find('a') + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/'+ch['href'])) + if i == 0: + self.story.setMetadata('datePublished', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat)) + if i == len(chapters)-1: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat)) + i=i+1 + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + info = soup.find('table', {'class' : 'story_info'}) + + # no convenient way to calculate word count as it is logged differently for stories with and without series + + labels = info.findAll('tr') + for tr in labels: + value = tr.find('td') + label = tr.find('th').string + + if 'Summary' in label: + self.setDescription(url,value) + + if 'Rating' in label: + self.story.setMetadata('rating', value.string) + + if 'Category' in label: + cats = value.findAll('a') + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = value.findAll('a') + for char in chars: + self.story.addToList('characters',char.string) + + if 'Pairing' in label: + ships = value.findAll('a') + for ship in ships: + self.story.addToList('ships',ship.string) + + if 'Genre' in label: + genres = value.findAll('a') + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = value.findAll('a') + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Status' in label: + if 'not completed' in value.text: + self.story.setMetadata('status', 'In-Progress') + else: + self.story.setMetadata('status', 'Completed') + + try: + # Find Series name from series URL. + a = soup.find('div',{'class' : 'sub_header'}) + series_name = a.find('a').string + i = a.text.split('#')[1] + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href']) + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'class' : 'story_text'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_netraptororg.py b/fanficdownloader/adapters/adapter_netraptororg.py new file mode 100644 index 0000000..6d4effb --- /dev/null +++ b/fanficdownloader/adapters/adapter_netraptororg.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return NetRaptorOrgAdapter + +class NetRaptorOrgAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','netrap') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%m/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'netraptor.org' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url+'&index=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + pagetitle = soup.find('div',{'id':'pagetitle'}) + a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/fanfiction/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url)) + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_nfacommunitycom.py b/fanficdownloader/adapters/adapter_nfacommunitycom.py new file mode 100644 index 0000000..4d428e7 --- /dev/null +++ b/fanficdownloader/adapters/adapter_nfacommunitycom.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# Search for XXX comments--that's where things are most likely to need changing. + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return NfaCommunityComAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class NfaCommunityComAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','nfa') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" # XXX + + @classmethod + def getAcceptDomains(cls): + return ['www.nfacommunity.com','nfacommunity.com'] + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'nfacommunity.com' # XXX + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + + # Furthermore, there's a couple sites now with more than + # one warning level for different ratings. And they're + # fussy about it. nfacommunity has two: 4 & 5. + # we'll try 5 first. + addurl = "&ageconsent=ok&warning=5" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + + # Since the warning text can change by warning level, let's + # look for the warning pass url. nfacommunity uses + # &warning= -- actually, so do other sites. Must be an + # eFiction book. + + # viewstory.php?sid=1882&warning=4 + # viewstory.php?sid=1654&ageconsent=ok&warning=5 + #print data + #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + ## Not all sites use Genre, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_nhamagicalworldsus.py b/fanficdownloader/adapters/adapter_nhamagicalworldsus.py new file mode 100644 index 0000000..bd541cc --- /dev/null +++ b/fanficdownloader/adapters/adapter_nhamagicalworldsus.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return NHAMagicalWorldsUsAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class NHAMagicalWorldsUsAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','nha') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = " %d/%m/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'nha.magical-worlds.us' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + try: + # in case link points somewhere other than the first chapter + a = soup.findAll('option')[1]['value'] + self.story.setMetadata('storyId',a.split('=',)[1]) + url = 'http://'+self.host+'/'+a + soup = bs.BeautifulSoup(self._fetchUrl(url)) + except: + pass + + for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}): + a = info.find('a') + if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \ + ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']: + self.story.setMetadata('title',stripHTML(a)) + break + + + # Find the chapters: + chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$')) + if len(chapters) == 0: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + for chapter in chapters: + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d): + try: + return d.name + except: + return "" + + cats = info.findAll('a',href=re.compile('categories.php')) + for cat in cats: + self.story.addToList('category',cat.string) + + a = info.find('a', href=re.compile(r'viewuser.php')) + val = a.nextSibling + svalue = "" + while not defaultGetattr(val) == 'br': + val = val.nextSibling + val = val.nextSibling + while not defaultGetattr(val) == 'br': + svalue += unicode(val) + val = val.nextSibling + self.setDescription(url,svalue) + + #does not provide convenient way to get word count + labels = info.findAll('i') + for labelspan in labels: + value = labelspan.nextSibling + label = stripHTML(labelspan) + + if 'Rating' in label: + self.story.setMetadata('rating', value.split(' -')[0]) + + if 'Genres' in label: + genres = value.string.split(', ') + for genre in genres: + if 'None' not in genre: + self.story.addToList('genre',genre.split(' -')[0]) + + if 'Characters' in label: + chars = value.string.split(', ') + for char in chars: + if 'None' not in char: + self.story.addToList('characters',char.split(' -')[0]) + + if 'Warnings' in label: + warnings = value.string.split(', ') + for warning in warnings: + if 'None' not in warning: + self.story.addToList('warnings',warning.split(' -')[0]) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat)) + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + + soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them + + story = soup.find('div', {"id" : "story"}) + + if None == story: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,story) diff --git a/fanficdownloader/adapters/adapter_nickandgregnet.py b/fanficdownloader/adapters/adapter_nickandgregnet.py new file mode 100644 index 0000000..cd0fd48 --- /dev/null +++ b/fanficdownloader/adapters/adapter_nickandgregnet.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return NickAndGregNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class NickAndGregNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','nag') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y/%m/%d" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.nickandgreg.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$" + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&i=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + chapters = soup.find('select') + for chapter in chapters.findAll('option'): + if chapter.text != 'Story Index' and chapter.text != 'Chapters': + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + for div in asoup.findAll('td', {'class' : 'tblborder6'}): + a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + if a != None: + break + + self.setDescription(url,div.find('br').nextSibling) + + a=div.text.split('Rating:') + if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0]) + + a=div.text.split('Characters:') + if len(a) == 2: + for char in a[1].split(' -')[0].split(', '): + self.story.addToList('characters',char) + + a=div.text.split('Genres:') + if len(a) == 2: + for genre in a[1].split(' -')[0].split(', '): + self.story.addToList('genre',genre) + + a=div.text.split('Warnings:') + if len(a) == 2: + for warn in a[1].split(' -')[0].split(', '): + if 'none' not in warn: + self.story.addToList('warnings',warn) + + a=div.text.split('Completed:') + if len(a) ==2: + if 'Yes' in a[1]: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + a=div.text.split('Published:') + if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat)) + + a=div.text.split('Updated:') + if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat)) + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + # wrap a div around it. + divsoup = bs.BeautifulStoneSoup('<div class="story"></div>', + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + div = divsoup.find('div') + div.append(soup.find('table', {'class' : 'tblborder6'})) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_nocturnallightnet.py b/fanficdownloader/adapters/adapter_nocturnallightnet.py new file mode 100644 index 0000000..214e38b --- /dev/null +++ b/fanficdownloader/adapters/adapter_nocturnallightnet.py @@ -0,0 +1,177 @@ +import re +import urllib2 +import urlparse + +from .. import BeautifulSoup + +from base_adapter import BaseSiteAdapter, makeDate +from .. import exceptions + + +def getClass(): + return NocturnalLightNetAdapter + + +# yields Tag _and_ NavigableString siblings from the given tag. The +# BeautifulSoup findNextSiblings() method for some reasons only returns either +# NavigableStrings _or_ Tag objects, not both. +def _yield_next_siblings(tag): + sibling = tag.nextSibling + while sibling: + yield sibling + sibling = sibling.nextSibling + + +class NocturnalLightNetAdapter(BaseSiteAdapter): + SITE_ABBREVIATION = 'nln' + SITE_DOMAIN = 'nocturnal-light.net' + BASE_URL = 'http://' + SITE_DOMAIN + '/fanfiction/' + STORY_URL_TEMPLATE = BASE_URL + 'story/%s' + AUTHORS_URL_TEMPLATE = BASE_URL + 'authors/%s' + + DATETIME_FORMAT = '%m-%d-%y' + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + url_tokens = self.parsedUrl.path.split('/') + story_id = url_tokens[url_tokens.index('story') + 1] + + self.story.setMetadata('storyId', story_id) + self._setURL(self.STORY_URL_TEMPLATE % story_id) + self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION) + + def _customized_fetch_url(self, url, exception=None, parameters=None): + if exception: + try: + data = self._fetchUrl(url, parameters) + except urllib2.HTTPError: + raise exception(self.url) + # Just let self._fetchUrl throw the exception, don't catch and + # customize it. + else: + data = self._fetchUrl(url, parameters) + + return BeautifulSoup.BeautifulSoup(data) + + @staticmethod + def getSiteDomain(): + return NocturnalLightNetAdapter.SITE_DOMAIN + + @classmethod + def getSiteExampleURLs(cls): + return cls.STORY_URL_TEMPLATE % 1234 + + def getSiteURLPattern(self): + return re.escape(self.STORY_URL_TEMPLATE[:-2]) + r'\d+.*$' + + def extractChapterUrlsAndMetadata(self): + soup = self._customized_fetch_url(self.url) + + # Since no 404 error code we have to raise the exception ourselves. + # A title that is just 'by' indicates that there is no author name + # and no story title available. + if soup.title.string.strip() == 'by': + raise exceptions.StoryDoesNotExist(self.url) + + # "storycontent" is found in a single-chapter story + author_anchor = soup.find('div', id=lambda id: id in ('main', 'storycontent')).h1.a + self.story.setMetadata('author', author_anchor.string) + + url_tokens = author_anchor['href'].split('/') + author_id = url_tokens[url_tokens.index('authors')+1] + self.story.setMetadata('authorId', author_id) + self.story.setMetadata('authorUrl', self.AUTHORS_URL_TEMPLATE % author_id) + + chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/')) + for chapter_anchor in chapter_anchors: + url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href']) + self.chapterUrls.append((chapter_anchor.string, url)) + + author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href']) + soup = self._customized_fetch_url(author_url) + story_id = self.story.getMetadata('storyId') + for listbox in soup('div', {'class': 'listbox'}): + url_tokens = listbox.a['href'].split('/') + # Found the div containing the story's metadata; break the loop and + # parse the element + if story_id == url_tokens[url_tokens.index('story')+1]: + break + else: + raise exceptions.FailedToDownload(self.url) + + title = listbox.a.string + self.story.setMetadata('title', title) + + # No chapter anchors found in the original story URL, so the story has + # only a single chapter. + if not chapter_anchors: + self.chapterUrls.append((title, self.url)) + + for b_tag in listbox('b'): + key = b_tag.string.strip(':') + try: + value = b_tag.nextSibling.string.replace('•', '').strip(': ') + # This can happen with some fancy markup in the summary. Just + # ignore this error and set value to None, the summary parsing + # takes care of this + except AttributeError: + value = None + + if key == 'Summary': + contents = [] + keep_summary_html = self.getConfig('keep_summary_html') + + for sibling in _yield_next_siblings(b_tag): + if isinstance(sibling, BeautifulSoup.Tag): + if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br': + break + + if keep_summary_html: + contents.append(self.utf8FromSoup(author_url, sibling)) + else: + contents.append(''.join(sibling(text=True))) + else: + contents.append(sibling) + + # Pop last break line tag + contents.pop() + self.story.setMetadata('description', ''.join(contents)) + + elif key == 'Category': + for sibling in b_tag.findNextSiblings(['a', 'b']): + if sibling.name == 'b': + break + + self.story.addToList('category', sibling.string) + + elif key == 'Rating': + self.story.setMetadata('rating', value) + + elif key == 'Chapters': + self.story.setMetadata('numChapters', int(value)) + + # Also parse reviews number which lies right after the chapters + # section + reviews_anchor = b_tag.findNextSibling('a') + reviews = reviews_anchor.string.split(' ')[1].strip('()') + self.story.setMetadata('reviews', reviews) + + elif key == 'Completed': + self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress') + + elif key == 'Date Added': + self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT)) + + elif key == 'Last Updated': + self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT)) + + elif key == 'Read': + self.story.setMetadata('readings', value.split()[0]) + + if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')): + raise exceptions.AdultCheckRequired(self.url) + + def getChapterText(self, url): + soup = self._customized_fetch_url(url) + return self.utf8FromSoup(url, soup.find('div', id='storytext')) diff --git a/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py b/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py new file mode 100644 index 0000000..87008a5 --- /dev/null +++ b/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return OcclumencySycophantHexComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class OcclumencySycophantHexComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','osph') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'occlumency.sycophanthex.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'This story contains adult content and/or themes.' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['rememberme'] = '1' + params['sid'] = '' + params['intent'] = '' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Logout" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + try: + # in case link points somewhere other than the first chapter + a = soup.findAll('option')[1]['value'] + self.story.setMetadata('storyId',a.split('=',)[1]) + url = 'http://'+self.host+'/'+a + soup = bs.BeautifulSoup(self._fetchUrl(url)) + except: + pass + + for info in asoup.findAll('table', {'class' : 'border'}): + a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + if a != None: + self.story.setMetadata('title',stripHTML(a)) + break + + + # Find the chapters: + chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')) + if len(chapters) == 0: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + for chapter in chapters: + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d): + try: + return d.name + except: + return "" + + cats = info.findAll('a',href=re.compile('categories.php')) + for cat in cats: + self.story.addToList('category',cat.string) + + + a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId'))) + val = a.nextSibling + svalue = "" + while not defaultGetattr(val) == 'br': + val = val.nextSibling + val = val.nextSibling + while not defaultGetattr(val) == 'table': + svalue += str(val) + val = val.nextSibling + self.setDescription(url,svalue) + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = info.findAll('b') + for labelspan in labels: + value = labelspan.nextSibling + label = stripHTML(labelspan) + + if 'Rating' in label: + self.story.setMetadata('rating', value) + + if 'Word Count' in label: + self.story.setMetadata('numWords', value) + + if 'Genres' in label: + genres = value.string.split(', ') + for genre in genres: + if genre != 'none': + self.story.addToList('genre',genre) + + if 'Characters' in label: + chars = value.string.split(', ') + for char in chars: + if char != 'none': + self.story.addToList('characters',char) + + if 'Warnings' in label: + warnings = value.string.split(', ') + for warning in warnings: + if warning != ' none': + self.story.addToList('warnings',warning) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + data = data.replace('<div align="left"', '<div align="left">') + + soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr')) + + story = soup.find('div', {"align" : "left"}) + + if None == story: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,story) diff --git a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py new file mode 100644 index 0000000..d4ae909 --- /dev/null +++ b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py @@ -0,0 +1,270 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return OneDirectionFanfictionComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class OneDirectionFanfictionComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','odf') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'onedirectionfanfiction.com' + + @classmethod + def getAcceptDomains(cls): + return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com'] + + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if "Age Consent Required" in data: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while value and not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_phoenixsongnet.py b/fanficdownloader/adapters/adapter_phoenixsongnet.py new file mode 100644 index 0000000..d8918d0 --- /dev/null +++ b/fanficdownloader/adapters/adapter_phoenixsongnet.py @@ -0,0 +1,240 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2, urllib, cookielib + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return PhoenixSongNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class PhoenixSongNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/') + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','phs') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%B %d %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.phoenixsong.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/fanfiction/story/1234/" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/fanfiction/story/")+r"\d+/?$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Please login to continue.' in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['txtusername'] = self.username + params['txtpassword'] = self.password + else: + params['txtusername'] = self.getConfig("username") + params['txtpassword'] = self.getConfig("password") + #params['remember'] = '1' + params['login'] = 'Login' + + loginUrl = 'http://' + self.getSiteDomain() + '/users/processlogin.php' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['txtusername'])) + d = self._fetchUrl(loginUrl, params) + + if 'Please login to continue.' in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['txtusername'])) + raise exceptions.FailedToLogin(url,params['txtusername']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + if self.getConfig('force_login'): + self.performLogin(url) + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + b = soup.find('div', {'id' : 'nav25'}) + a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. /fanfiction/stories.php?psid=125 + a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + chapters = soup.find('select') + if chapters == None: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + for b in soup.findAll('b'): + if b.text == "Updated": + date = b.nextSibling.string.split(': ')[1].split(',') + self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat)) + else: + i = 0 + chapters = chapters.findAll('option') + for chapter in chapters: + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['value'])) + if i == 0: + self.story.setMetadata('storyId',chapter['value'].split('/')[3]) + head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b') + for b in head: + if b.text == "Updated": + date = b.nextSibling.string.split(': ')[1].split(',') + self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat)) + + if i == (len(chapters)-1): + head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b') + for b in head: + if b.text == "Updated": + date = b.nextSibling.string.split(': ')[1].split(',') + self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat)) + i = i+1 + + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + info = asoup.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$")) + while info != None: + info = info.findNext('div') + b = info.find('b') + val = b.nextSibling + + if 'Rating' in b.string: + self.story.setMetadata('rating', val.string.split(': ')[1]) + + if 'Words' in b.string: + self.story.setMetadata('numWords', val.string.split(': ')[1]) + + if 'Setting' in b.string: + self.story.addToList('category', val.string.split(': ')[1]) + + if 'Status' in b.string: + if 'Completed' in val: + val = 'Completed' + else: + val = 'In-Progress' + self.story.setMetadata('status', val) + + if 'Summary' in b.string: + b.extract() + info.find('br').extract() + self.setDescription(url,info) + break + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + chapter=bs.BeautifulSoup('<div class="story"></div>') + for p in soup.findAll('p'): + if "This is for problems with the formatting or the layout of the chapter." in stripHTML(p): + break + chapter.append(p) + + for a in chapter.findAll('div'): + a.extract() + for a in chapter.findAll('table'): + a.extract() + for a in chapter.findAll('script'): + a.extract() + for a in chapter.findAll('form'): + a.extract() + for a in chapter.findAll('textarea'): + a.extract() + + + if None == chapter: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,chapter) diff --git a/fanficdownloader/adapters/adapter_pommedesangcom.py b/fanficdownloader/adapters/adapter_pommedesangcom.py new file mode 100644 index 0000000..4d553a0 --- /dev/null +++ b/fanficdownloader/adapters/adapter_pommedesangcom.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return PommeDeSangComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class PommeDeSangComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # pommedesang.com has two 'sections', shown in URL as + # 'efiction' and 'sds' that change how things should be + # handled. + # http://pommedesang.com/efiction/viewstory.php?sid=1234 + # http://pommedesang.com/sds/viewstory.php?sid=1234 + self.section=self.parsedUrl.path.split('/',)[1] + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','pmds') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + if 'efiction' in self.section: + self.dateformat = "%b %d, %Y" + else: + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'pommedesang.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/sds/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return r"http://"+self.getSiteDomain()+"/(efiction|sds)?/viewstory.php\?sid=\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/'+self.section+'/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=5" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+')) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + +# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+self.section+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile('viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']: + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py b/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py new file mode 100644 index 0000000..eceb332 --- /dev/null +++ b/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return PonyFictionArchiveNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class PonyFictionArchiveNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + # normalized story URL. + if "explicit" in self.parsedUrl.netloc: + self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + self.dateformat = "%d/%b/%y" + else: + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + self.dateformat = "%d %b %Y" + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','pffa') + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'ponyfictionarchive.net' + + @classmethod + def getAcceptDomains(cls): + return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&warning=9" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + self.story.setMetadata('status',status.string) + + section = soup.findAll('span', {'class' : 'General'})[1] + + self.story.setMetadata('rating', section.previousSibling.previousSibling.string) + + value = section.nextSibling + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_portkeyorg.py b/fanficdownloader/adapters/adapter_portkeyorg.py new file mode 100644 index 0000000..0a8e417 --- /dev/null +++ b/fanficdownloader/adapters/adapter_portkeyorg.py @@ -0,0 +1,282 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 +import cookielib as cl + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# Search for XXX comments--that's where things are most likely to need changing. + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return PortkeyOrgAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class PortkeyOrgAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','prtky') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%m/%y" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'fanfiction.portkey.org' # XXX + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/story/1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/story/")+r"\d+(/\d+)?$" + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + logger.debug("URL: "+url) + + # portkey screws around with using a different URL to set the + # cookie and it's a pain. So... cheat! + if self.is_adult or self.getConfig("is_adult"): + cookie = cl.Cookie(version=0, name='verify17', value='1', + port=None, port_specified=False, + domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False, + path='/', path_specified=True, + secure=False, + expires=time.time()+10000, + discard=False, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False) + self.cookiejar.set_cookie(cookie) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "You must be over 18 years of age to view it" in data: # XXX + raise exceptions.AdultCheckRequired(self.url) + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + #print data + + # Now go hunting for all the meta data and the chapter list. + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"/profile/\d+")) + #print("======a:%s"%a) + self.story.setMetadata('authorId',a['href'].split('/')[-1]) + self.story.setMetadata('authorUrl','http://'+self.host+a['href']) + self.story.setMetadata('author',a.string) + + ## Going to get the rest from the author page. + authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + ## Title + titlea = authsoup.find('a', href=re.compile(r'/story/'+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(titlea)) + metablock = titlea.parent + + # Find the chapters: + for chapter in soup.find('select',{'name':'select5'}).findAll('option', {'value':re.compile(r'/story/'+self.story.getMetadata('storyId')+"/\d+$")}): + # just in case there's tags, like <i> in chapter titles. + chtitle = stripHTML(chapter) + if not chtitle: + chtitle = "(Untitled Chapter)" + self.chapterUrls.append((chtitle,'http://'+self.host+chapter['value'])) + + if len(self.chapterUrls) == 0: + self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + # <SPAN class="dark-small-bold">Contents:</SPAN> <SPAN class="small-grey">NC17 </SPAN> + # <SPAN class="dark-small-bold">Published: </SPAN><SPAN class="small-grey">12/11/07</SPAN> + # <SPAN class="dark-small-bold"><BR> + # Description:</SPAN> <SPAN class="small-black">A special book helps Harry tap into the power the Dark Lord knows not. Of course it’s a book on sex magic and rituals… but Harry’s not complaining. Spurned on by the ghost of a pervert founder, Harry leads his friends in the hunt for Voldemort’s Horcruxes. + # EROTIC COMEDY! Loads of crude humor and sexual situations! + # </SPAN> + labels = metablock.findAll('span',{'class':'dark-small-bold'}) + for labelspan in labels: + value = labelspan.findNext('span').string + label = stripHTML(labelspan) +# print("\nlabel:%s\nlabel:%s\nvalue:%s\n"%(labelspan,label,value)) + + if 'Description' in label: + self.setDescription(url,value) + + if 'Contents' in label: + self.story.setMetadata('rating', value) + + if 'Words' in label: + self.story.setMetadata('numWords', value) + + # if 'Categories' in label: + # cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + # catstext = [cat.string for cat in cats] + # for cat in catstext: + # self.story.addToList('category',cat.string) + + # if 'Characters' in label: + # chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + # charstext = [char.string for char in chars] + # for char in charstext: + # self.story.addToList('characters',char.string) + + if 'Genre' in label: + # genre is typo'ed on the site--it falls between the + # dark-small-bold label and dark-small-bold content + # spans. + svalue = "" + value = labelspan.nextSibling + while not defaultGetattr(value,'class') == 'dark-small-bold': + svalue += str(value) + value = value.nextSibling + + for genre in svalue.split("/"): + genre = genre.strip() + if genre != 'None': + self.story.addToList('genre',genre) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + # if 'Warnings' in label: + # warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + # warningstext = [warning.string for warning in warnings] + # self.warning = ', '.join(warningstext) + # for warning in warningstext: + # self.story.addToList('warnings',warning.string) + + if 'Status' in label: + if 'Completed' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + # try: + # # Find Series name from series URL. + # a = metablock.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + # series_name = a.string + # series_url = 'http://'+self.host+'/'+a['href'] + + # # use BeautifulSoup HTML parser to make everything easier to find. + # seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + # i=1 + # for a in storyas: + # if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + # self.setSeries(series_name, i) + # break + # i+=1 + # except: + # # I find it hard to care if the series parsing fails + # pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + data = self._fetchUrl(url) + + data = data.replace("HTML>","div>") + + soup = bs.BeautifulSoup(data) + + #print("soup:%s"%soup) + tag = soup.find('td', {'class' : 'story'}) + if tag == None and "<center><b>Chapter does not exist!</b></center>" in data: + print("Chapter is missing at: %s"%url) + return self.utf8FromSoup(url,bs.BeautifulStoneSoup("<div><p><center><b>Chapter does not exist!</b></center></p><p>Chapter is missing at: <a href='%s'>%s</a></p></div>"%(url,url))) + tag.name='div' # force to be a div to avoid problems with nook. + + centers = tag.findAll('center') + # first two and last two center tags are some script, 'report + # story', 'report story' and an ad. + centers[0].extract() + centers[1].extract() + centers[-1].extract() + centers[-2].extract() + + if None == tag: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,tag) diff --git a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py new file mode 100644 index 0000000..da9e845 --- /dev/null +++ b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','pns') + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId')) + + + @staticmethod + def getSiteDomain(): + return 'www.potionsandsnitches.net' + + @classmethod + def getAcceptDomains(cls): + return ['www.potionsandsnitches.net','potionsandsnitches.net'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$" + + def extractChapterUrlsAndMetadata(self): + + url = self.url+'&index=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + ## <meta name='description' content='<p>Description</p> ...' > + ## Summary, strangely, is in the content attr of a <meta name='description'> tag + ## which is escaped HTML. Unfortunately, we can't use it because they don't + ## escape (') chars in the desc, breakin the tag. + #meta_desc = soup.find('meta',{'name':'description'}) + #metasoup = bs.BeautifulStoneSoup(meta_desc['content']) + #self.story.setMetadata('description',stripHTML(metasoup)) + + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next div class='listbox' + svalue = "" + while not defaultGetattr(value,'class') == 'listbox': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + if "Snape and Harry (required)" in char: + self.story.addToList('characters',"Snape") + self.story.addToList('characters',"Harry") + else: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class')) + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + # limit date values, there's some extra chars. + self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y")) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y")) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/fanfiction/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) + +def getClass(): + return PotionsAndSnitchesNetSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_potterficscom.py b/fanficdownloader/adapters/adapter_potterficscom.py new file mode 100644 index 0000000..c837927 --- /dev/null +++ b/fanficdownloader/adapters/adapter_potterficscom.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import datetime +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return PotterFicsComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class PotterFicsComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + + # normalized story URL. gets rid of chapter if there, left with chapter index URL + nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId') + self._setURL(nurl) + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','potficscom') + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.potterfics.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://www.potterfics.com/historias/12345 http://www.potterfics.com/historias/12345/capitulo-1 " + + def getSiteURLPattern(self): + #http://www.potterfics.com/historias/127583 + #http://www.potterfics.com/historias/127583/capitulo-1 + #http://www.potterfics.com/historias/127583/capitulo-4 + #http://www.potterfics.com/historias/92810 -> Complete story + #http://www.potterfics.com/historias/111194 -> Complete, single chap + p = re.escape("http://"+self.getSiteDomain()+"/historias/")+\ + r"(?P<id>\d+)(/capitulo-(?P<ch>\d+))?/?$" + return p + + def needToLoginCheck(self, data): + # partials used to avoid having to figure out what was wrong + # with included utf8 higher chars. + if 'Para ver esta historia, por favor inicia tu sesi' in data \ + or '<script>alert("El nombre de usuario o contrase' in data: + return True + else: + return False + + def performLogin(self,url): + params = {} + + if self.password: + params['login_usuario'] = self.username + params['login_password'] = self.password + else: + params['login_usuario'] = self.getConfig("username") + params['login_password'] = self.getConfig("password") + params['login_ck'] = '1' + + loginUrl = 'http://www.potterfics.com/secciones/usuarios/login.php' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['login_usuario'])) + d = self._postUrl(loginUrl,params) + + #print("d:%s"%d) + if '<script>alert("El nombre de usuario o contrase' in d: + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['login_usuario'])) + raise exceptions.FailedToLogin(url,params['login_usuario']) + return False + else: + return True + + def extractChapterUrlsAndMetadata(self): + + #this converts '/historias/12345' to 'http://www.potterfics.com/historias/12345' + def makeAbsoluteURL(url): + if url[0] == '/': + url = 'http://'+self.getSiteDomain()+url + return url + + #use this to get month numbers from Spanish months + SpanishMonths = { + 'enero' : '01', + 'febrero' : '02', + 'marzo' : '03', + 'abril' : '04', + 'mayo' : '05', + 'junio' : '06', + 'julio' : '07', + 'agosto' : '08', + 'septiembre' : '09', + 'octubre' : '10', + 'noviembre' : '11', + 'diciembre' : '12' + } + + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + #print data + + #deal with adult content login + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url,usecache=False) + + #set constant meta for this site: + #Set Language = Spanish + self.story.setMetadata('language', 'Spanish') + #Set Category = Harry Potter + # This is better done in plugin-defaults.ini and defaults.ini + # by adding a section for this site with the line: + # extracategories:Harry Potter + #self.story.addToList('category','Harry Potter') + + #get the rest of the meta + # use BeautifulSoup HTML parser to make everything easier to find. + #self closing br and img present! + soup = bs.BeautifulSoup(data,selfClosingTags=('br','img')) + + #we want the second table directly under the body, contains all the metadata + table = soup.html.body.findAll('table', recursive=False)[1] + #within that, we want the second row, first cell + cell = table.tr.findNextSibling('tr').td + + #find first metadata block--isn't first if logged in + mb = cell.div.findNextSibling('div',{'align':'left'}) + #Get meta... + self.story.setMetadata('title', stripHTML(mb.b)) + #strip out brackets on rating + self.story.setMetadata('rating', mb.span.string[1:-1]) + #Completion status is denoted by the presence of this image: + if mb.find('img',title="Historia terminada"): + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + #find next metadata block + #author details + mb = mb.findNextSibling('div') + self.story.setMetadata('author', mb.b.a.string.strip()) + self.story.setMetadata('authorUrl', makeAbsoluteURL(mb.b.a['href'])) + self.story.setMetadata('authorId', self.story.getMetadata('authorUrl').split('/')[4]) + #dates and times + mb = mb.find('span') + #posted/published = Escrita + date = mb.find(text=re.compile('Escrita el ')).strip().split() + year = int(date[7][:-1]) # need to remove the last char from year, it is a comma + month = int(SpanishMonths[date[5].lower()]) + day = int(date[3]) + time = date[8].split(':') + hour = int(time[0]) + minute = int(time[1]) + self.story.setMetadata('datePublished', datetime.datetime(year, month, day, hour, minute)) + #updated = Actualizada + date = mb.find(text=re.compile('Actualizada el ')).strip().split() + year = int(date[7][:-1]) # need to remove the last char from year, it is a comma + month = int(SpanishMonths[date[5].lower()]) + day = int(date[3]) + time = date[8].split(':') + hour = int(time[0]) + minute = int(time[1]) + self.story.setMetadata('dateUpdated', datetime.datetime(year, month, day, hour, minute)) + + mb = mb.span.findNextSibling('span').findNextSibling('span') + wc = mb.find(text=re.compile(' palabras en total')).strip() + self.story.setMetadata('numWords', wc.split()[0]) + + #then we come to categories and genres. Oh dear. On this site, categories hold everything from genre, to ships, to crossovers. + #To make things worse, there is also another genre field, which often holds similar/duplicate info. Links to genre pages do not work + #though, so perhaps those will be phased out? + #for now, put them all into the genre list + links = mb.findAll('a',href=re.compile('/(categorias|generos)/\d+')) + genlist = [i.string.strip() for i in links] + self.story.extendList('genre',genlist) + + #get the chapter urls + #we can go back to the table cell we found before + #get its last element and work backwards to find the last ordered list on the page + list = cell.contents[len(cell)-1].findPrevious('ol') + chapters = [] + revs = 0 + chnum = 0 + for li in list: + chnum += 1 + chTitle = str(chnum) + '. ' + li.a.b.string.strip() + chURL = makeAbsoluteURL(li.a['href']) + chapters.append((chTitle,chURL)) + #Get reviews, add to total + revs += int(li.div.a.string.split()[0]) + + self.chapterUrls.extend(chapters) + self.story.setMetadata('numChapters', len(chapters)) + self.story.setMetadata('reviews', revs) + + #Now for the description... this may be tricky... + #if it is there (doesn't have to be), it will be before the chapter list, + #separated by a horizontal rule, and after the google ad bar + + #get list's parent div + mb = list.parent + #get the div before that, will either be the description, or the google ad bar + mb = mb.findPreviousSibling('div') + if 'google_ad_client' in str(mb): + #couldn't find description, leaving it blank + pass + else: + self.setDescription(url,mb) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr','img')) + + div = soup.find('div', {'id' : 'cuerpoHistoria'}) + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_potterheadsanonymouscom.py b/fanficdownloader/adapters/adapter_potterheadsanonymouscom.py new file mode 100644 index 0000000..c8f061c --- /dev/null +++ b/fanficdownloader/adapters/adapter_potterheadsanonymouscom.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return PotterHeadsAnonymousComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class PotterHeadsAnonymousComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','pha') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %b %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'fanfic.potterheadsanonymous.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # Since the warning text can change by warning level, let's + # look for the warning pass url. ksarchive uses + # &warning= -- actually, so do other sites. Must be an + # eFiction book. + + # viewstory.php?sid=1882&warning=4 + # viewstory.php?sid=1654&ageconsent=ok&warning=5 + #print data + #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + pagetitle = soup.find('div',{'id':'pagetitle'}) + + ## Title + a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_pretendercentrecom.py b/fanficdownloader/adapters/adapter_pretendercentrecom.py new file mode 100644 index 0000000..5e9590a --- /dev/null +++ b/fanficdownloader/adapters/adapter_pretendercentrecom.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return PretenderCenterComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class PretenderCenterComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','ptdc') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%m/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'pretendercentre.com' + + @classmethod + def getAcceptDomains(cls): + return ['www.pretendercentre.com','pretendercentre.com'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/missingpieces/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/missingpieces/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/missingpieces/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.findAll('div', {'id' : 'story'})[1] + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_psychficcom.py b/fanficdownloader/adapters/adapter_psychficcom.py new file mode 100644 index 0000000..fa53815 --- /dev/null +++ b/fanficdownloader/adapters/adapter_psychficcom.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return PsychFicComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class PsychFicComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','psyf') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%B %d, %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.psychfic.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.text + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_qafficcom.py b/fanficdownloader/adapters/adapter_qafficcom.py new file mode 100644 index 0000000..c60c330 --- /dev/null +++ b/fanficdownloader/adapters/adapter_qafficcom.py @@ -0,0 +1,265 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return QafFicComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class QafFicComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','atp') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.qaf-fic.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/atp/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=")+r"\d+$" + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&warning=NC-17" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\s+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title and author + a = soup.find('div', {'id' : 'pagetitle'}) + + aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',aut['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/atp/'+aut['href']) + self.story.setMetadata('author',aut.string) + aut.extract() + + self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)]) + + # Find the chapters: + chapters=soup.find('select') + if chapters != None: + for chapter in chapters.findAll('option'): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/atp/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])) + else: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}): + a = list.find('a') + if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']: + break + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = list.findAll('span', {'class' : 'classification'}) + for labelspan in labels: + label = labelspan.string + value = labelspan.nextSibling + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'classification' and value != None: + if "Featured Stories" not in value: + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value[:len(value)-2]) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + for char in value.string.split(', '): + if not 'None' in char: + self.story.addToList('characters',char) + + if 'Genre' in label: + for genre in value.string.split(', '): + if not 'None' in genre: + self.story.addToList('genre',genre) + + if 'Warnings' in label: + for warning in value.string.split(', '): + if not 'None' in warning: + self.story.addToList('warnings',warning) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + if list.find('a', href=re.compile(r"series.php")) != None: + for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")): + # Find Series name from series URL. + series_url = 'http://'+self.host+'/atp/'+series['href'] + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + name=seriessoup.find('div', {'id' : 'pagetitle'}) + name.find('a').extract() + self.setSeries(name.text.split(' by[')[0], i) + self.story.setMetadata('seriesUrl',series_url) + i=0 + break + i+=1 + if i == 0: + break + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_restrictedsectionorg.py b/fanficdownloader/adapters/adapter_restrictedsectionorg.py new file mode 100644 index 0000000..f03b4e3 --- /dev/null +++ b/fanficdownloader/adapters/adapter_restrictedsectionorg.py @@ -0,0 +1,265 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 +import cookielib as cl +from datetime import datetime + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return RestrictedSectionOrgSiteAdapter + +class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + + # normalized story URL. + # get story/file and storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/' + m.group('filestory') + '.php?' + m.group('filestory') + '=' + self.story.getMetadata('storyId')) + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + self.story.setMetadata('siteabbrev','ressec') + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %b %Y" # 20 Nov 2005 + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + return 'www.restrictedsection.org' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/story.php?story=1234 http://"+cls.getSiteDomain()+"/file.php?file=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain())+r"/(?P<filestory>file|story).php\?(file|story)=(?P<id>\d+)$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + logger.debug("URL: "+url) + + # one-shot stories use file url instead of story. 'Luckily', + # we don't have to worry about one-shots becoming + # multi-chapter because ressec is frozen. Still need 'story' + # url for metadata, however. + try: + if 'file' in url: + data = self._postUrlUP(url) + soup = bs.BeautifulSoup(data) + storya = soup.find('a',href=re.compile(r"^story.php\?story=\d+")) + url = 'http://'+self.host+'/'+storya['href'].split('&')[0] # strip rs_session + + fileas = soup.find('a',href=re.compile(r"^file.php\?file=\d+")) + if fileas: + for filea in fileas: + if 'Previous Chapter' in filea.string or 'Next Chapter' in filea.string: + raise exceptions.FailedToDownload(self.getSiteDomain() +" Cannot use chapter url with multi-chapter stories on this site.") + + logger.debug("metadata URL: "+url) + data = self._fetchUrl(url) + # print data + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + if "Story not found" in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Story not found.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + # check user/pass on a chapter for multi-chapter + if 'file' not in self.url: + self._postUrlUP('http://'+self.host+'/'+soup.find('a', href=re.compile(r"^file.php\?file=\d+"))['href']) + + ## Title + h2 = soup.find('h2') + + # Find authorid and URL from... author url. + a = h2.find('a') + ahref = a['href'].split('&')[0] # strip rs_session + + self.story.setMetadata('authorId',ahref.split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref) + self.story.setMetadata('author',stripHTML(a)) + + # title, remove byauthorname. + auth=stripHTML(a) + title=stripHTML(h2) + self.story.setMetadata('title',title[:title.index(" by "+auth)]) + + dates = soup.findAll('span', {'class':'date'}) + if dates: # only for multi-chapter + self.story.setMetadata('datePublished', makeDate(stripHTML(dates[0]), self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(stripHTML(dates[-1]), self.dateformat)) + + words = soup.findAll('span', {'class':'size'}) + wordcount=0 + for w in words: + wordcount = wordcount + int(w.string[:-6].replace(',','')) + + self.story.setMetadata('numWords',"%s"%wordcount) + + self.story.setMetadata('rating', soup.find('a',href=re.compile(r"^rating.php\?rating=\d+")).string) + + # other tags + + labels = soup.find('table', {'class':'info'}).findAll('th') + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if label != None: + + if 'Categories' in label: + for g in stripHTML(value).split('\n'): + self.story.addToList('genre',g) + + if 'Pairings' in label: + for g in stripHTML(value).split('\n'): + self.story.addToList('ships',g) + + if 'Summary' in label: + self.setDescription(url,stripHTML(value).replace("\n"," ").replace("\r","")) + value.extract() # remove summary incase it contains file URLs. + + if 'Updated' in label: # one-shots only. + print "value:%s"%value + value.find('sup').extract() # remove 'st', 'nd', 'th' ordinals + print "value:%s"%value + date = makeDate(stripHTML(value), '%d %B %Y') # full month name + self.story.setMetadata('datePublished', date) + + if 'Length' in label: # one-shots only. + self.story.setMetadata('numWords',value.string[:-6]) + + # one-shot. + if 'file' in self.url: + self.chapterUrls.append((self.story.getMetadata('title'),self.url)) + else: # multi-chapter + # Find the chapters: 'library_storyview.php?chapterid=3 + chapters=soup.findAll('a', href=re.compile(r"^file.php\?file=\d+")) + if len(chapters)==0: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No chapters found.") + else: + for chapter in chapters: + chhref = chapter['href'].split('&')[0] # strip rs_session + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chhref)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + + + def _postUrlUP(self, url): + params = {} + if self.password: + params['username'] = self.username + params['password'] = self.password + else: + params['username'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['accept.x'] = 1 + params['accept.y'] = 1 + + excpt=None + for sleeptime in [0.5, 1.5, 4, 9]: + time.sleep(sleeptime) + try: + data = self._postUrl(url, params) + if data == "Unable to connect to the database": + raise exceptions.FailedToDownload("Site reported 'Unable to connect to the database'") + if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data: + raise exceptions.FailedToLogin(url,params['username']) + return data + except exceptions.FailedToLogin, ftl: + # no need to retry these. + raise(ftl) + except Exception, e: + excpt=e + logger.warn("Caught an exception reading URL: %s Exception %s."%(unicode(url),unicode(e))) + + logger.error("Giving up on %s" %url) + logger.exception(excpt) + raise(excpt) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._postUrlUP(url) + #print("data:%s"%data) + + # some stories have html that confuses the parser. For story + # text we don't care about anything before '<table id="page"' + # and seems to clear the issue. + data = data[data.index('<table id="page"'):] + + soup = bs.BeautifulSoup(data) + + div = soup.find('td',{'id':'page_content'}) + div.name='div' + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + ## Remove stuff from page_content + + # Remove all tags before the first <hr> after class=info table (including hr) + hr = div.find('table',{'class':'info'}).findNext('hr') + for tag in hr.findAllPrevious(): + tag.extract() + hr.extract() + + # Remove all tags after the last <hr> (including hr) + hr = div.findAll('hr')[-1] + for tag in hr.findAllNext(): + tag.extract() + hr.extract() + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_samandjacknet.py b/fanficdownloader/adapters/adapter_samandjacknet.py new file mode 100644 index 0000000..dbfa209 --- /dev/null +++ b/fanficdownloader/adapters/adapter_samandjacknet.py @@ -0,0 +1,342 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# By virtue of being recent and requiring both is_adult and user/pass, +# adapter_fanficcastletvnet.py is the best choice for learning to +# write adapters--especially for sites that use the eFiction system. +# Most sites that have ".../viewstory.php?sid=123" in the story URL +# are eFiction. + +# For non-eFiction sites, it can be considerably more complex, but +# this is still a good starting point. + +# In general an 'adapter' needs to do these five things: + +# - 'Register' correctly with the downloader +# - Site Login (if needed) +# - 'Are you adult?' check (if needed--some do one, some the other, some both) +# - Grab the chapter list +# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page) +# - Grab the chapter texts + +# Search for XXX comments--that's where things are most likely to need changing. + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return SamAndJackNetAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SamAndJackNetAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','sjn') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%b %d, %Y" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'samandjack.net' # XXX + + @classmethod + def getSiteExampleURLs(self): + return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/fanfics/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + + # Furthermore, there's a couple sites now with more than + # one warning level for different ratings. And they're + # fussy about it. midnightwhispers has three: 10, 3 & 5. + # we'll try 5 first. + addurl = "&ageconsent=ok&warning=5" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + + # Since the warning text can change by warning level, let's + # look for the warning pass url. nfacommunity uses + # &warning= -- actually, so do other sites. Must be an + # eFiction book. + + # viewstory.php?sid=1882&warning=4 + # viewstory.php?sid=1654&ageconsent=ok&warning=5 + #print data + #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + pagetitle = soup.find('div',{'id':'pagetitle'}) + ## Title + a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + # (fetch multiple authors) + alist = soup.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+")) + for a in alist: + self.story.addToList('authorId',a['href'].split('=')[1]) + self.story.addToList('authorUrl','http://'+self.host+'/fanfics/'+a['href']) + self.story.addToList('author',a.string) + + # Reviews + reviewdata = soup.find('div', {'id' : 'sort'}) + a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one. + self.story.setMetadata('reviews',stripHTML(a)) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfics/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + self.setDescription(url,value) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + ## Not all sites use Genre, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + value=value.replace(' | ','') + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/fanfics/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_samdeanarchivenu.py b/fanficdownloader/adapters/adapter_samdeanarchivenu.py new file mode 100644 index 0000000..398a2bc --- /dev/null +++ b/fanficdownloader/adapters/adapter_samdeanarchivenu.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return SamDeanArchiveNuAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SamDeanArchiveNuAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','sda') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'samdean.archive.nu' + + @classmethod + def getAcceptDomains(cls): + return ['www.samdean.archive.nu','samdean.archive.nu'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+r"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title and author + a = soup.find('div', {'id' : 'pagetitle'}) + + aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',aut['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href']) + self.story.setMetadata('author',aut.string) + aut.extract() + + self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)]) + + # Find the chapters: + chapters=soup.find('select') + if chapters != None: + for chapter in chapters.findAll('option'): + # just in case there's tags, like <i> in chapter titles. http://samdean.archive.nu/viewstory.php?sid=4317&chapter=2 + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value'])) + else: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}): + a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + if a != None: + break + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = list.findAll('span', {'class' : 'classification'}) + for labelspan in labels: + label = labelspan.string + value = labelspan.nextSibling + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'classification': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value[:len(value)-2]) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + for char in value.string.split(', '): + if not 'None' in char: + self.story.addToList('characters',char) + + if 'Genre' in label: + for genre in value.string.split(', '): + if not 'None' in genre: + self.story.addToList('genre',genre) + + if 'Warnings' in label: + for warning in value.string.split(', '): + if not 'None' in warning: + self.story.addToList('warnings',warning) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_scarheadnet.py b/fanficdownloader/adapters/adapter_scarheadnet.py new file mode 100644 index 0000000..8ba6044 --- /dev/null +++ b/fanficdownloader/adapters/adapter_scarheadnet.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return ScarHeadNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class ScarHeadNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','shn') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%m/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'scarhead.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=5" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # Since the warning text can change by warning level, let's + # look for the warning pass url. ksarchive uses + # &warning= -- actually, so do other sites. Must be an + # eFiction book. + + # viewstory.php?sid=1882&warning=4 + # viewstory.php?sid=1654&ageconsent=ok&warning=5 + #print data + #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data) + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + pagetitle = soup.find('tr',{'valign':'top'}) + + ## Title + a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + if '/' == cat.string[0]: + self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0]) + elif 'Harry' in cat.string: + self.story.addToList('ships',cat.string.split('(')[0]) + else: + self.story.addToList('category',cat.string) + if '(' in cat.string: + self.story.addToList('category',cat.string.split('(')[1].split(')')[0]) + + + + + chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for genre in genres: + self.story.addToList('genre',genre.string) + + warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + textsoup = stripHTML(soup) + + a = textsoup.split('Published: ')[1].split(' ')[0] + self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat)) + a = textsoup.split('Updated: ')[1].split(' ')[0] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat)) + a = textsoup.split('Rating: ')[1].split(' ')[0] + self.story.setMetadata('rating', a) + a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0] + self.story.setMetadata('numWords', a) + a = textsoup.split('Completed: ')[1].split(' ')[0] + if 'Yes' in a: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + #a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0] + #self.setDescription(url,a) + + + + a=soup.find(text=re.compile("Summary: ")) + i=0 + svalue = "" + while i == 0: + try: + b = str(a) + svalue += b.split('Summary: ')[1] + except: + svalue += str(a) + if a.nextSibling != None: + a = a.nextSibling + else: + a = a.parent.nextSibling + if 'Disclaimer: ' in stripHTML(a): + i=1 + self.setDescription(url,svalue) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py b/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py new file mode 100644 index 0000000..f90a77e --- /dev/null +++ b/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return ScarvesAndCoffeeNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','scacf') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.scarvesandcoffee.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=20" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('genre',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_sg1heliopoliscom.py b/fanficdownloader/adapters/adapter_sg1heliopoliscom.py new file mode 100644 index 0000000..f36edae --- /dev/null +++ b/fanficdownloader/adapters/adapter_sg1heliopoliscom.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return SG1HeliopolisComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SG1HeliopolisComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + self.section=self.parsedUrl.path.split('/',)[1] + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','sghp') + + # If all stories from the site fall into the same category, + # the site itself isn't likely to label them as such, so we + # do. Can't use extracategories, could be Atlantis or SG-1 + if 'atlantis' in self.section: + self.story.addToList("category","Stargate: Atlantis") + else: + self.story.addToList("category","Stargate: SG-1") + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y.%m.%d" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'sg1-heliopolis.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/atlantis/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return r"http://sg1-heliopolis.com/(archive|adult|atlantis)?/viewstory.php\?sid=\d+$" + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+self.section+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_sheppardweircom.py b/fanficdownloader/adapters/adapter_sheppardweircom.py new file mode 100644 index 0000000..ddce071 --- /dev/null +++ b/fanficdownloader/adapters/adapter_sheppardweircom.py @@ -0,0 +1,317 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# By virtue of being recent and requiring both is_adult and user/pass, +# adapter_fanficcastletvnet.py is the best choice for learning to +# write adapters--especially for sites that use the eFiction system. +# Most sites that have ".../viewstory.php?sid=123" in the story URL +# are eFiction. + +# For non-eFiction sites, it can be considerably more complex, but +# this is still a good starting point. + +# In general an 'adapter' needs to do these five things: + +# - 'Register' correctly with the downloader +# - Site Login (if needed) +# - 'Are you adult?' check (if needed--some do one, some the other, some both) +# - Grab the chapter list +# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page) +# - Grab the chapter texts + +# Search for XXX comments--that's where things are most likely to need changing. + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return SheppardWeirComAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SheppardWeirComAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','swf') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%B %d, %Y" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'sheppardweir.com' # XXX + + @classmethod + def getSiteExampleURLs(self): + return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/fanfics/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if "Age Consent Required" in data: # XXX + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + pagetitle = soup.find('div',{'id':'pagetitle'}) + ## Title + a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + # (fetch multiple authors) + alist = soup.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+")) + for a in alist: + self.story.addToList('authorId',a['href'].split('=')[1]) + self.story.addToList('authorUrl','http://'+self.host+'/fanfics/'+a['href']) + self.story.addToList('author',a.string) + + + # Reviews + reviewdata = soup.find('div', {'id' : 'sort'}) + a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one. + self.story.setMetadata('reviews',stripHTML(a)) + + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfics/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # Summary + summarydata = unicode(soup.find('div',{'class':'content'})) + start='<span class="label">Summary: </span>' + end='</div>' + summarydata = summarydata[summarydata.index(start)+len(start):summarydata.rindex(end)] + self.setDescription(url,bs.BeautifulSoup(summarydata)) + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + ## Not all sites use Genre, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + value=value.replace(' - ','') + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/fanfics/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_simplyundeniablecom.py b/fanficdownloader/adapters/adapter_simplyundeniablecom.py new file mode 100644 index 0000000..131b8be --- /dev/null +++ b/fanficdownloader/adapters/adapter_simplyundeniablecom.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return SimplyUndeniableComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SimplyUndeniableComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','sud') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.simplyundeniable.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Please log in now' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "My Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + if "Please log in now" in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: You need to have access to the restricted section.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('h1') + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + for info in asoup.findAll('table', {'cellpadding' : '5'}): + a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + if a != None: + self.story.setMetadata('title',stripHTML(a)) + break + + # Find the chapters: + if "Disclaimer" in data: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + a = info.find('td', {'valign' : 'top'}).find('p') + self.setDescription(url,a) + + a = info.find('td', {'class' : 'greysm'}).findAll('b') + self.story.setMetadata('datePublished', makeDate(stripHTML(a[0].nextSibling), self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].nextSibling), self.dateformat)) + if 'Yes' in a[2].nextSibling: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + self.story.setMetadata('rating', a[3].nextSibling) + self.story.setMetadata('numWords', a[4].nextSibling) + + + warnings = info.find('td', {'width' : '45'}).nextSibling.nextSibling.text.split(', ') + for warning in warnings: + if 'none' not in warning: + self.story.addToList('warnings',warning) + chars = info.find('td', {'width' : '51'}).nextSibling.nextSibling.text.split(', ') + for char in chars: + if '&' in char: + self.story.addToList('ships',char) + else: + self.story.addToList('characters',char) + genres = info.find('td', {'width' : '36'}).nextSibling.nextSibling.text.split(', ') + for genre in genres: + self.story.addToList('genre',genre) + + cat = info.find('a', href=re.compile(r'categories.php')) + self.story.addToList('category',cat.string) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('span', {'class' : 'style'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + div.name='div' + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_sinfuldesireorg.py b/fanficdownloader/adapters/adapter_sinfuldesireorg.py new file mode 100644 index 0000000..2187c68 --- /dev/null +++ b/fanficdownloader/adapters/adapter_sinfuldesireorg.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return SinfulDesireOrgAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SinfulDesireOrgAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','snds') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.sinful-desire.org' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&warning=5" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/archive/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_siyecouk.py b/fanficdownloader/adapters/adapter_siyecouk.py new file mode 100644 index 0000000..9129bfc --- /dev/null +++ b/fanficdownloader/adapters/adapter_siyecouk.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return SiyeCoUkAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SiyeCoUkAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8",]# 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','siye') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y.%m.%d" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.siye.co.uk' # XXX + + @classmethod + def getAcceptDomains(cls): + return ['www.siye.co.uk','siye.co.uk'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/siye/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+r"(www\.)?siye\.co\.uk/(siye/)?"+re.escape("viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + # Except it doesn't this time. :-/ + url = self.url #+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/siye/'+a['href']) + self.story.setMetadata('author',a.string) + + # need(or easier) to pull other metadata from the author's list page. + authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + ## Title + titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(titlea)) + + # Find the chapters (from soup, not authsoup): + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href'])) + + if self.chapterUrls: + self.story.setMetadata('numChapters',len(self.chapterUrls)) + else: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + self.story.setMetadata('numChapters',1) + + # The stuff we can get from the chapter list/one-shot page are + # in the first table with 95% width. + metatable = soup.find('table',{'width':'95%'}) + + # Categories + cat_as = metatable.findAll('a', href=re.compile(r'categories.php')) + for cat_a in cat_as: + self.story.addToList('category',stripHTML(cat_a)) + + moremetaparts = stripHTML(metatable).split('\n') + for part in moremetaparts: + part = part.strip() + if part.startswith("Characters:"): + part = part[part.find(':')+1:] + for item in part.split(','): + if item.strip() == "Harry/Ginny": + self.story.addToList('characters',"Harry") + self.story.addToList('characters',"Ginny") + elif item.strip() not in ("None","All"): + self.story.addToList('characters',item) + + if part.startswith("Genres:"): + part = part[part.find(':')+1:] + for item in part.split(','): + if item.strip() != "None": + self.story.addToList('genre',item) + + if part.startswith("Warnings:"): + part = part[part.find(':')+1:] + for item in part.split(','): + if item.strip() != "None": + self.story.addToList('warnings',item) + + if part.startswith("Rating:"): + part = part[part.find(':')+1:] + self.story.setMetadata('rating',part) + + if part.startswith("Summary:"): + part = part[part.find(':')+1:] + self.setDescription(url,part) + #self.story.setMetadata('description',part) + + # want to get the next tr of the table. + #print("%s"%titlea.parent.parent.findNextSibling('tr')) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # SIYE formats stories in the author list differently when their part of a series. + # Look for non-series... + divdesc = titlea.parent.parent.find('div',{'class':'desc'}) + if not divdesc: + # ... now look for series. + divdesc = titlea.parent.parent.findNextSibling('tr').find('div',{'class':'desc'}) + + moremeta = stripHTML(divdesc) + #print("moremeta:%s"%moremeta) + for part in moremeta.replace(' - ','\n').split('\n'): + #print("part:%s"%part) + try: + (name,value) = part.split(': ') + except: + # not going to worry about fancier processing for the bits + # that don't match. + continue + name=name.strip() + value=value.strip() + if name == 'Published': + self.story.setMetadata('datePublished', makeDate(value, self.dateformat)) + if name == 'Updated': + self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat)) + if name == 'Completed': + if value == 'Yes': + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + if name == 'Words': + self.story.setMetadata('numWords', value) + + try: + # Find Series name from series URL. + a = titlea.findPrevious('a', href=re.compile(r"series.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + # soup = bs.BeautifulSoup(self._fetchUrl(url)) + # BeautifulSoup objects to <p> inside <span>, which + # technically isn't allowed. + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + # not the most unique thing in the world, but it appears to be + # the best we can do here. + story = soup.find('span', {'style' : 'font-size: 100%;'}) + + if None == story: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + story.name='div' + + return self.utf8FromSoup(url,story) diff --git a/fanficdownloader/adapters/adapter_spikeluvercom.py b/fanficdownloader/adapters/adapter_spikeluvercom.py new file mode 100644 index 0000000..511463d --- /dev/null +++ b/fanficdownloader/adapters/adapter_spikeluvercom.py @@ -0,0 +1,208 @@ +# Software: eFiction +import re +import urllib2 +import urlparse + +from .. import BeautifulSoup +from ..htmlcleanup import stripHTML + +from base_adapter import BaseSiteAdapter, makeDate +from .. import exceptions + + +def getClass(): + return SpikeluverComAdapter + + +# yields Tag _and_ NavigableString siblings from the given tag. The +# BeautifulSoup findNextSiblings() method for some reasons only returns either +# NavigableStrings _or_ Tag objects, not both. +def _yield_next_siblings(tag): + sibling = tag.nextSibling + while sibling: + yield sibling + sibling = sibling.nextSibling + + +class SpikeluverComAdapter(BaseSiteAdapter): + SITE_ABBREVIATION = 'slc' + SITE_DOMAIN = 'spikeluver.com' + + BASE_URL = 'http://' + SITE_DOMAIN + '/SpuffyRealm/' + LOGIN_URL = BASE_URL + 'user.php?action=login' + VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d' + METADATA_URL_SUFFIX = '&index=1' + AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=5' + + DATETIME_FORMAT = '%m/%d/%Y' + STORY_DOES_NOT_EXIST_ERROR_TEXT = 'That story does not exist on this archive. You may search for it or return to the home page.' + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + query_data = urlparse.parse_qs(self.parsedUrl.query) + story_id = query_data['sid'][0] + + self.story.setMetadata('storyId', story_id) + self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id)) + self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION) + + def _customized_fetch_url(self, url, exception=None, parameters=None): + if exception: + try: + data = self._fetchUrl(url, parameters) + except urllib2.HTTPError: + raise exception(self.url) + # Just let self._fetchUrl throw the exception, don't catch and + # customize it. + else: + data = self._fetchUrl(url, parameters) + + return BeautifulSoup.BeautifulSoup(data) + + @staticmethod + def getSiteDomain(): + return SpikeluverComAdapter.SITE_DOMAIN + + @classmethod + def getSiteExampleURLs(cls): + return cls.VIEW_STORY_URL_TEMPLATE % 1234 + + def getSiteURLPattern(self): + return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$' + + def extractChapterUrlsAndMetadata(self): + soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX) + + errortext_div = soup.find('div', {'class': 'errortext'}) + if errortext_div: + error_text = ''.join(errortext_div(text=True)).strip() + if error_text == self.STORY_DOES_NOT_EXIST_ERROR_TEXT: + raise exceptions.StoryDoesNotExist(self.url) + + # No additional login is required, just check for adult + pagetitle_div = soup.find('div', id='pagetitle') + if pagetitle_div.a['href'].startswith('javascript:'): + if not(self.is_adult or self.getConfig('is_adult')): + raise exceptions.AdultCheckRequired(self.url) + + url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX]) + soup = self._customized_fetch_url(url) + + pagetitle_div = soup.find('div', id='pagetitle') + self.story.setMetadata('title', stripHTML(pagetitle_div.a)) + + author_anchor = pagetitle_div.a.findNextSibling('a') + url = urlparse.urljoin(self.BASE_URL, author_anchor['href']) + components = urlparse.urlparse(url) + query_data = urlparse.parse_qs(components.query) + + self.story.setMetadata('author', stripHTML(author_anchor)) + self.story.setMetadata('authorId', query_data['uid']) + self.story.setMetadata('authorUrl', url) + + sort_div = soup.find('div', id='sort') + self.story.setMetadata('reviews', stripHTML(sort_div('a')[1])) + + listbox_tag = soup.find('div', {'class': 'listbox'}) + for span_tag in listbox_tag('span'): + key = span_tag.string.strip(' :') + try: + value = stripHTML(span_tag.nextSibling) + # This can happen with some fancy markup in the summary. Just + # ignore this error and set value to None, the summary parsing + # takes care of this + except AttributeError: + value = None + + if key == 'Summary': + contents = [] + keep_summary_html = self.getConfig('keep_summary_html') + + for sibling in _yield_next_siblings(span_tag): + if isinstance(sibling, BeautifulSoup.Tag): + # Encountered next label, break. Not as bad as other + # e-fiction sites, let's hope this is enough for proper + # parsing. + if sibling.name == 'span' and sibling.get('class', None) == 'label': + break + + if keep_summary_html: + contents.append(self.utf8FromSoup(self.url, sibling)) + else: + contents.append(''.join(sibling(text=True))) + else: + contents.append(sibling) + + # Remove the preceding break line tag and other crud + contents.pop() + contents.pop() + self.story.setMetadata('description', ''.join(contents)) + + elif key == 'Rated': + self.story.setMetadata('rating', value) + + elif key == 'Categories': + for sibling in span_tag.findNextSiblings(['a', 'br']): + if sibling.name == 'br': + break + + self.story.addToList('category', stripHTML(sibling)) + + # Seems to be always "None" for some reason + elif key == 'Characters': + for sibling in span_tag.findNextSiblings(['a', 'br']): + if sibling.name == 'br': + break + self.story.addToList('characters', stripHTML(sibling)) + + elif key == 'Genres': + for sibling in span_tag.findNextSiblings(['a', 'br']): + if sibling.name == 'br': + break + + self.story.addToList('genre', stripHTML(sibling)) + + elif key == 'Warnings': + for sibling in span_tag.findNextSiblings(['a', 'br']): + if sibling.name == 'br': + break + self.story.addToList('warnings', stripHTML(sibling)) + + # Challenges + + elif key == 'Series': + a = span_tag.findNextSibling('a') + if not a: + continue + self.story.setMetadata('series', stripHTML(a)) + self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href'])) + + elif key == 'Chapters': + self.story.setMetadata('numChapters', int(value)) + + elif key == 'Completed': + self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress') + + elif key == 'Word count': + self.story.setMetadata('numWords', value) + + elif key == 'Published': + self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT)) + + elif key == 'Updated': + self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT)) + + for p_tag in listbox_tag.findNextSiblings('p'): + chapter_anchor = p_tag.find('a', href=lambda href: href and href.startswith('viewstory.php?sid=')) + if not chapter_anchor: + continue + + title = stripHTML(chapter_anchor) + url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href']) + self.chapterUrls.append((title, url)) + + def getChapterText(self, url): + url += self.AGE_CONSENT_URL_SUFFIX + soup = self._customized_fetch_url(url) + return self.utf8FromSoup(url, soup.find('div', id='story')) diff --git a/fanficdownloader/adapters/adapter_squidgeorgpeja.py b/fanficdownloader/adapters/adapter_squidgeorgpeja.py new file mode 100644 index 0000000..98927ca --- /dev/null +++ b/fanficdownloader/adapters/adapter_squidgeorgpeja.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + + +def getClass(): + return SquidgeOrgPejaAdapter + +## XXX IMPORTANT NOTE!! This adapter is for squidge.org/peja ONLY! +## There are lots of other sites and stuff under squidge.org that +## we're not supporting. If/When we ever want to support more +## sections of squidge.org, FFDL will need to be changed more +## fundamentally to find different adapters under the same domain. +## +## For now, I've only implemented the part for ini section names so +## if/when more adapters under squidge.org come along, existing ini +## files will still work correctly. + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class SquidgeOrgPejaAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["utf8", + "Windows-1252"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('https://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','wwomb') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.squidge.org' + + @classmethod # must be @classmethod, don't remove it. + def getConfigSection(cls): + # The config section name. Only override if != site domain. + return cls.getSiteDomain()+'/peja' + + @classmethod + def getSiteExampleURLs(cls): + return "https://"+cls.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + # but not https://www.squidge.org/peja/cgi-bin/viewstory.php?sid=47746 -- that's the 'Site Map' negative look aead + return r"https?"+re.escape("://"+self.getSiteDomain()+"/")+r"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"(?!47746)\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + data = self._fetchUrl(url) + + if "fatal MySQL error was encountered" in data: + raise exceptions.FailedToDownload("Site SQL Error--bad story") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + # Find authorid and URL from... author url. + author = soup.find('div', {'id':"pagetitle"}).find('a') + self.story.setMetadata('authorId',author['href'].split('=')[1]) + self.story.setMetadata('authorUrl','https://'+self.host+'/peja/cgi-bin/'+author['href']) + self.story.setMetadata('author',author.string) + + authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')}) + self.story.setMetadata('title',stripHTML(title)) + titleblock=title.parent.parent + + chapterselect=soup.find('select',{'name':'chapter'}) + if chapterselect: + for ch in chapterselect.findAll('option'): + self.chapterUrls.append((stripHTML(ch),'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value'])) + else: + self.chapterUrls.append((title,url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="classification">Rated:</span> NC-17<br /> etc + labels = titleblock.findAll('span',{'class':'classification'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + #print("label:%s value:%s"%(label,value)) + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'classification': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + if value.endswith("["): + value = value[:-1] + self.story.setMetadata('rating', value) + + if 'Characters' in label: + for char in value.split(','): + self.story.addToList('characters',char.strip()) + + if 'Genre' in label: + for genre in value.split(','): + if genre.strip() != "None": + self.story.addToList('genre',genre.strip()) + + if 'Warnings' in label: + for warning in value.split(','): + if warning.strip() != 'None': + self.story.addToList('warnings',warning.strip()) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Fandoms' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + + # site has started including a tracking script between Updated label and date--sometimes... + svalue = "" + while value is not None and not defaultGetattr(value,'class') == 'classification': + if '<script' not in unicode(value): + svalue += unicode(value) + value = value.nextSibling + #print("svalue:%s"%svalue) + + self.story.setMetadata('dateUpdated', makeDate(stripHTML(svalue), self.dateformat)) + + try: + # Find Series name from series URL. + # http://www.squidge.org/peja/cgi-bin/series.php?seriesid=254 + a = titleblock.find('a', href=re.compile(r"series.php\?seriesid=\d+")) + series_name = a.string + series_url = 'https://'+self.host+'/peja/cgi-bin/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + # don't count the 'site map' story. See the url pattern method. + if '47746' not in a['href']: + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + chaptext = soup.find('div',{'id':"story"}).find('span') + + if None == chaptext: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,chaptext) diff --git a/fanficdownloader/adapters/adapter_stargateatlantisorg.py b/fanficdownloader/adapters/adapter_stargateatlantisorg.py new file mode 100644 index 0000000..f842853 --- /dev/null +++ b/fanficdownloader/adapters/adapter_stargateatlantisorg.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return StargateAtlantisOrgAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class StargateAtlantisOrgAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','stat') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%B %d %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'stargate-atlantis.org' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/fanfics/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title and author + a = soup.find('div', {'id' : 'pagetitle'}) + + aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',aut['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/fanfics/'+aut['href']) + self.story.setMetadata('author',aut.string) + aut.extract() + + self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)]) + + # Find the chapters: + chapters=soup.findAll('div', {'class' : 'content'}) + if len(chapters) > 1: + for chapter in chapters: + # just in case there's tags, like <i> in chapter titles. + link = chapter.find('a') + self.chapterUrls.append((stripHTML(link),'http://'+self.host+'/fanfics/'+link['href'])) + else: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}): + a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + if a != None: + break + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = list.findAll('span', {'class' : 'classification'}) + for labelspan in labels: + label = labelspan.string + value = labelspan.nextSibling + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'tail' and value != None: + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value[:len(value)-2]) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + for char in value.string.split(', '): + if not 'None' in char: + self.story.addToList('characters',char) + + if 'Genre' in label: + for genre in value.string.split(', '): + if not 'None' in genre: + self.story.addToList('genre',genre) + + if 'Warnings' in label: + for warning in value.string.split(', '): + if not 'None' in warning: + self.story.addToList('warnings',warning) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/fanfics/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_storiesofardacom.py b/fanficdownloader/adapters/adapter_storiesofardacom.py new file mode 100644 index 0000000..5ad3ef8 --- /dev/null +++ b/fanficdownloader/adapters/adapter_storiesofardacom.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return StoriesOfArdaComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class StoriesOfArdaComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','soa') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.storiesofarda.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/chapterlistview.asp?SID=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title and author + a = soup.find('th', {'colspan' : '3'}) + + aut = a.find('a') + self.story.setMetadata('authorId',aut['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href']) + self.story.setMetadata('author',aut.string) + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + a.find('em').extract() + self.story.setMetadata('title',stripHTML(a)) + + # Find the chapters: chapterview.asp?sid=7000&cid=30919 + chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$")) + if len(chapters)==1: + self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href'])) + else: + for chapter in chapters: + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + summary = soup.find('td', {'colspan' : '3'}) + self.setDescription(url,summary) + + # no convenient way to get word count + + for td in asoup.findAll('td', {'colspan' : '3'}): + if td.find('a', href=re.compile('chapterlistview.asp\?SID='+self.story.getMetadata('storyId'))) != None: + break + td=td.nextSibling.nextSibling + self.story.setMetadata('dateUpdated', makeDate(stripHTML(td).split(': ')[1], self.dateformat)) + tr=td.parent.nextSibling.nextSibling.nextSibling.nextSibling + td=tr.findAll('td') + self.story.setMetadata('rating', td[0].string.split(': ')[1]) + self.story.setMetadata('status', td[2].string.split(': ')[1]) + self.story.setMetadata('datePublished', makeDate(stripHTML(td[4]).split(': ')[1], self.dateformat)) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + if self.getConfig('is_adult'): + params = {'confirmAge':'1'} + data = self._postUrl(url,params) + else: + data = self._fetchUrl(url) + + data = data[data.index('<table width="90%" align="center">'):] + data.replace("<body","<notbody").replace("<BODY","<NOTBODY") + + soup = bs.BeautifulStoneSoup(data, + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + if "Please indicate that you are an adult by selecting the appropriate choice below" in data: + raise exceptions.FailedToDownload("Chapter requires you be an adult. Set is_adult in personal.ini (chapter url:%s)" % url) + + div = soup.find('table', {'width' : '90%'}).find('td') + div.name='div' + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_storiesonlinenet.py b/fanficdownloader/adapters/adapter_storiesonlinenet.py new file mode 100644 index 0000000..2858aa1 --- /dev/null +++ b/fanficdownloader/adapters/adapter_storiesonlinenet.py @@ -0,0 +1,422 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return StoriesOnlineNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class StoriesOnlineNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url + self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0]) + if 'storyInfo' in self.story.getMetadata('storyId'): + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','strol') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y-%m-%d" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'storiesonline.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/s/1234 http://"+cls.getSiteDomain()+"/s/1234:4010" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)?" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if self.needToLogin \ + or 'Free Registration' in data \ + or "Invalid Password!" in data \ + or "Invalid User Name!" in data \ + or "Log In" in data \ + or "Access to unlinked chapters requires" in data: + self.needToLogin = True + return self.needToLogin + + def performLogin(self, url): + params = {} + + if self.password: + params['theusername'] = self.username + params['thepassword'] = self.password + else: + params['theusername'] = self.getConfig("username") + params['thepassword'] = self.getConfig("password") + params['rememberMe'] = '1' + params['page'] = 'http://'+self.getSiteDomain()+'/' + params['submit'] = 'Login' + + loginUrl = 'http://' + self.getSiteDomain() + '/login.php' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['theusername'])) + + d = self._fetchUrl(loginUrl, params,usecache=False) + + if "My Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['theusername'])) + raise exceptions.FailedToLogin(url,params['theusername']) + return False + else: + return True + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + self.needToLogin = False + try: + data = self._fetchUrl(url+":i") + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + elif e.code == 401: + self.needToLogin = True + data = '' + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url+":i",usecache=False) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + #print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('h1') + self.story.setMetadata('title',stripHTML(a)) + + notice = soup.find('div', {'class' : 'notice'}) + if notice: + self.story.setMetadata('notice',unicode(notice)) + + # Find authorid and URL from... author url. + for a in soup.findAll('a', href=re.compile(r"/a/\w+")): + self.story.addToList('authorId',a['href'].split('/')[2]) + self.story.addToList('authorUrl','http://'+self.host+a['href']) + self.story.addToList('author',stripHTML(a).replace("'s Page","")) + + # Find the chapters: + chapters = soup.findAll('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId')+":\d+$")) + if len(chapters) != 0: + for chapter in chapters: + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href'])) + else: + self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId'))) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # surprisingly, the detailed page does not give enough details, so go to author's page + page=0 + i=0 + while i == 0: + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page))) + + a = asoup.findAll('td', {'class' : 'lc2'}) + for lc2 in a: + if lc2.find('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId'))): + i=1 + break + if a[len(a)-1] == lc2: + page=page+1 + + for cat in lc2.findAll('div', {'class' : 'typediv'}): + self.story.addToList('genre',cat.text) + + # in lieu of word count. + self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text) + + score = lc2.findNext('th', {'class' : 'num'}).text + if score != '-': + self.story.setMetadata('score', score) + + lc4 = lc2.findNext('td', {'class' : 'lc4'}) + desc = lc4.contents[0] + + try: + a = lc4.find('a', href=re.compile(r"/series/\d+/.*")) + if a: + # if there's a number after the series name, series_contents is a two element list: + # [<a href="...">Title</a>, u' (2)'] + series_contents = a.parent.contents + i = 0 if len(series_contents) == 1 else series_contents[1].strip(' ()') + seriesUrl = 'http://'+self.host+a['href'] + self.story.setMetadata('seriesUrl',seriesUrl) + series_name = stripHTML(a) + logger.debug("Series name= %s" % series_name) + series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl)) + if series_soup: + logger.debug("Retrieving Series - looking for name") + series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0] + logger.debug("Series name: '{0}'".format(series_name)) + self.setSeries(series_name, i) + desc = lc4.contents[2] + # Check if series is in a universe + universe_url = self.story.getList('authorUrl')[0] + "&type=uni" + universes_soup = bs.BeautifulSoup(self._fetchUrl(universe_url) ) + logger.debug("Universe url='{0}'".format(universe_url)) + if universes_soup: + universes = universes_soup.findAll('div', {'class' : 'ser-box'}) + logger.debug("Number of Universes: %d" % len(universes)) + for universe in universes: + logger.debug("universe.find('a')={0}".format(universe.find('a'))) + # The universe id is in an "a" tag that has an id but nothing else. It is the first tag. + # The id is prefixed with the letter "u". + universe_id = universe.find('a')['id'][1:] + logger.debug("universe_id='%s'" % universe_id) + universe_name = universe.find('div', {'class' : 'ser-name'}).text.partition(' ')[2] + logger.debug("universe_name='%s'" % universe_name) + # If there is link to the story, we have the right universe + story_a = universe.find('a', href=re.compile('/s/'+self.story.getMetadata('storyId'))) + if story_a: + logger.debug("Story is in a series that is in a universe! The universe is '%s'" % universe_name) + self.story.setMetadata("universe", universe_name) + self.story.setMetadata('universeUrl','http://'+self.host+ '/library/universe.php?id=' + universe_id) + break + else: + logger.debug("No universe page") + except: + pass + try: + a = lc4.find('a', href=re.compile(r"/universe/\d+/.*")) + logger.debug("Looking for universe - a='{0}'".format(a)) + if a: + self.story.setMetadata("universe",stripHTML(a)) + desc = lc4.contents[2] + # Assumed only one universe, but it does have a URL--use universeHTML + universe_name = stripHTML(a) + universeUrl = 'http://'+self.host+a['href'] + logger.debug("Retrieving Universe - about to get page - universeUrl='{0}".format(universeUrl)) + universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl)) + logger.debug("Retrieving Universe - have page") + if universe_soup: + logger.debug("Retrieving Universe - looking for name") + universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition(' —')[0] + logger.debug("Universes name: '{0}'".format(universe_name)) + + self.story.setMetadata('universeUrl',universeUrl) + logger.debug("Setting universe name: '{0}'".format(universe_name)) + self.story.setMetadata('universe',universe_name) + if self.getConfig("universe_as_series"): + self.setSeries(universe_name, 0) + self.story.setMetadata('seriesUrl',universeUrl) + else: + logger.debug("Do not have a universe") + except: + pass + + self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc) + + for b in lc4.findAll('b'): + #logger.debug('Getting metadata: "%s"' % b) + label = b.text + if label in ['Posted:', 'Concluded:', 'Updated:']: + value = b.findNext('noscript').text + #logger.debug('Have a date field label: "%s", value: "%s"' % (label, value)) + else: + value = b.nextSibling + #logger.debug('label: "%s", value: "%s"' % (label, value)) + + if 'Sex' in label: + self.story.setMetadata('rating', value) + + if 'Tags' in label or 'Codes' in label: + for code in re.split(r'\s*,\s*', value.strip()): + self.story.addToList('sitetags',code) + + if 'Posted' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + if 'Concluded' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) +# + status = lc4.find('span', {'class' : 'ab'}) + if status != None: + self.story.setMetadata('status', 'In-Progress') + if "Last Activity" in status.text: + # date is passed as a timestamp and converted in JS. + value = status.findNext('noscript').text + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + else: + self.story.setMetadata('status', 'Completed') + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + # some big chapters are split over several pages + pager = div.find('span', {'class' : 'pager'}) + if pager != None: + a = pager.previousSibling + while a != None: + logger.debug("before pager: {0}".format(a)) + b = a.previousSibling + a.extract() + a = b + + urls=pager.findAll('a') + urls=urls[:len(urls)-1] + pager.extract() + div.contents = div.contents[2:] +# logger.debug(div) + + for ur in urls: + soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div1 = soup.find('div', {'id' : 'story'}) + + # Find the "Continues" marker on the current page and remove everything after that. + continues = div.find('span', {'class' : 'conTag'}) + if continues != None: + while continues != None: +# logger.debug("removing end: {0}".format(continues)) + b = continues.nextSibling + continues.extract() + continues = b + + # Find the "Continued" marker and delete everything before that + continued = div1.find('span', {'class' : 'conTag'}) + if continued != None: + a = continued.previousSibling + while a != None: +# logger.debug("before conTag: {0}".format(a)) + b = a.previousSibling + a.extract() + a = b + # Remove the pager from the end if this is the last page + endPager = div1.find('span', {'class' : 'pager'}) + if endPager != None: + b = endPager.nextSibling + while endPager != None: + logger.debug("removing end: {0}".format(endPager)) + b = endPager.nextSibling + endPager.extract() + endPager = b + div1.contents = div1.contents[:len(div1) - 2] +# logger.debug("after removing pager: {0}".format(div1)) + for tag in div1.contents[2:]: + div.append(tag) + + # If it is a chapter, there are dates at the start for when it was posted or modified. These plus + # everything before them can be discarded. + postedDates = div.findAll('div', {'class' : 'date'}) + if postedDates: + a = postedDates[0].previousSibling + while a != None: +# logger.debug("before dates: {0}".format(a)) + b = a.previousSibling + a.extract() + a = b + for a in div.findAll('div', {'class' : 'date'}): + a.extract() + + # For single chapter stories, there is a copyright statement. Remove this and everything + # before it. + copy = div.find('h4', {'class': 'copy'}) + while copy != None: +# logger.debug("before copyright: {0}".format(copy)) + b = copy.previousSibling + copy.extract() + copy = b + + # For a story or the last chapter, remove voting form and the in library box + a = div.find('div', {'id' : 'vote-form'}) + if a != None: + a.extract() + a = div.find('div', {'id' : 'b-man-div'}) + if a != None: + a.extract() + + # Kill the "The End" header and everything after it. + a = div.find(['h2', 'h3'], {'class' : 'end'}) + logger.debug("Chapter end= '{0}'".format(a)) + while a != None: + b = a.nextSibling + a.extract() + a=b + + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py new file mode 100644 index 0000000..634adfe --- /dev/null +++ b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py @@ -0,0 +1,257 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class TenhawkPresentsComSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','thpc') + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + self.dateformat = "%b %d %Y" + + + @staticmethod + def getSiteDomain(): + return 'fanfiction.tenhawkpresents.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + addurl = "&ageconsent=ok&warning=3" + else: + addurl="" + + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + addurl = "&ageconsent=ok&warning=4" + url = self.url+'&index=1'+addurl + logger.debug("Changing URL: "+url) + self.performLogin(url) + data = self._fetchUrl(url,usecache=False) + + if "This story contains mature content which may include violence, sexual situations, and coarse language" in data: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId'))) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class')) + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + span = soup.find('div', {'id' : 'story'}) + + if None == span: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,span) + +def getClass(): + return TenhawkPresentsComSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py new file mode 100644 index 0000000..4bf8525 --- /dev/null +++ b/fanficdownloader/adapters/adapter_test1.py @@ -0,0 +1,367 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import datetime +import time +import logging +logger = logging.getLogger(__name__) + +from .. import exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class TestSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','tst1') + self.crazystring = u" crazy tests:[bare amp(&) quote(') amp(&) gt(>) lt(<) ATnT(AT&T) pound(£)]" + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + self.username='' + self.is_adult=False + + @staticmethod + def getSiteDomain(): + return 'test1.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"?sid=1234" + + def getSiteURLPattern(self): + return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$' + + def extractChapterUrlsAndMetadata(self): + idstr = self.story.getMetadata('storyId') + idnum = int(idstr) + self.do_sleep() + + if idnum >= 1000: + logger.warn("storyId:%s - Custom INI data will be used."%idstr) + + sections = ['teststory:%s'%idstr,'teststory:defaults'] + #print("self.get_config_list(sections,'valid_entries'):%s"%self.get_config_list(sections,'valid_entries')) + for key in self.get_config_list(sections,'valid_entries'): + if key.endswith("_list"): + nkey = key[:-len("_list")] + #print("addList:%s"%(nkey)) + for val in self.get_config_list(sections,key): + #print("addList:%s->%s"%(nkey,val)) + self.story.addToList(nkey,val.decode('utf-8').replace('{{storyId}}',idstr)) + else: + # Special cases: + if key in ['datePublished','dateUpdated']: + self.story.setMetadata(key,makeDate(self.get_config(sections,key),"%Y-%m-%d")) + else: + self.story.setMetadata(key,self.get_config(sections,key).decode('utf-8').replace('{{storyId}}',idstr)) + #print("set:%s->%s"%(key,self.story.getMetadata(key))) + + self.chapterUrls = [] + for (j,chap) in enumerate(self.get_config_list(sections,'chaptertitles'),start=1): + self.chapterUrls.append( (chap,self.url+"&chapter=%d"%j) ) + # self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"), + # ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"), + # ] + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + return + + if idstr == '665' and not (self.is_adult or self.getConfig("is_adult")): + logger.warn("self.is_adult:%s"%self.is_adult) + raise exceptions.AdultCheckRequired(self.url) + + if idstr == '666': + raise exceptions.StoryDoesNotExist(self.url) + + if idstr.startswith('670'): + time.sleep(1.0) + + if idstr.startswith('671'): + time.sleep(1.0) + + if self.getConfig("username"): + self.username = self.getConfig("username") + + if idstr == '668' and self.username != "Me" : + raise exceptions.FailedToLogin(self.url,self.username) + + if idstr == '664': + self.story.setMetadata(u'title',"Test Story Title "+idstr+self.crazystring) + self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)') + else: + self.story.setMetadata(u'title',"Test Story Title "+idstr) + self.story.setMetadata('author','Test Author aa') + self.story.setMetadata('storyUrl',self.url) + self.setDescription(self.url,u'Description '+self.crazystring+u''' Done +<p> +Some more longer description. "I suck at summaries!" "Better than it sounds!" "My first fic" +''') + self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d")) + if idstr == '669': + self.story.setMetadata('dateUpdated',datetime.datetime.now()) + else: + self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d")) + self.story.setMetadata('numWords','123456') + + if idnum % 2 == 1: + self.story.setMetadata('status','In-Progress') + else: + self.story.setMetadata('status','Completed') + + # greater than 10, no language or series. + if idnum < 10: + langs = { + 0:"English", + 1:"Russian", + 2:"French", + 3:"German", + } + self.story.setMetadata('language',langs[idnum%len(langs)]) + self.setSeries('The Great Test',idnum) + self.story.setMetadata('seriesUrl','http://test1.com?seriesid=1') + if idnum == 0: + self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum) + self.story.setMetadata('seriesUrl','http://test1.com?seriesid=0') + + self.story.setMetadata('rating','Tweenie') + + if idstr == '673': + self.story.addToList('author','Author From List 1') + self.story.addToList('author','Author From List 2') + self.story.addToList('author','Author From List 3') + self.story.addToList('author','Author From List 4') + self.story.addToList('author','Author From List 5') + self.story.addToList('author','Author From List 6') + self.story.addToList('author','Author From List 7') + self.story.addToList('author','Author From List 8') + self.story.addToList('author','Author From List 9') + self.story.addToList('author','Author From List 0') + self.story.addToList('author','Author From List q') + self.story.addToList('author','Author From List w') + self.story.addToList('author','Author From List e') + self.story.addToList('author','Author From List r') + self.story.addToList('author','Author From List t') + self.story.addToList('author','Author From List y') + self.story.addToList('author','Author From List u') + self.story.addToList('author','Author From List i') + self.story.addToList('author','Author From List o') + + self.story.addToList('authorId','98765-1') + self.story.addToList('authorId','98765-2') + self.story.addToList('authorId','98765-3') + self.story.addToList('authorId','98765-4') + self.story.addToList('authorId','98765-5') + self.story.addToList('authorId','98765-6') + self.story.addToList('authorId','98765-7') + self.story.addToList('authorId','98765-8') + self.story.addToList('authorId','98765-9') + self.story.addToList('authorId','98765-0') + self.story.addToList('authorId','98765-q') + self.story.addToList('authorId','98765-w') + self.story.addToList('authorId','98765-e') + self.story.addToList('authorId','98765-r') + self.story.addToList('authorId','98765-t') + self.story.addToList('authorId','98765-y') + self.story.addToList('authorId','98765-u') + self.story.addToList('authorId','98765-i') + self.story.addToList('authorId','98765-o') + + self.story.addToList('authorUrl','http://author/url-1') + self.story.addToList('authorUrl','http://author/url-2') + self.story.addToList('authorUrl','http://author/url-3') + self.story.addToList('authorUrl','http://author/url-4') + self.story.addToList('authorUrl','http://author/url-5') + self.story.addToList('authorUrl','http://author/url-6') + self.story.addToList('authorUrl','http://author/url-7') + self.story.addToList('authorUrl','http://author/url-8') + self.story.addToList('authorUrl','http://author/url-9') + self.story.addToList('authorUrl','http://author/url-0') + self.story.addToList('authorUrl','http://author/url-q') + self.story.addToList('authorUrl','http://author/url-w') + self.story.addToList('authorUrl','http://author/url-e') + self.story.addToList('authorUrl','http://author/url-r') + self.story.addToList('authorUrl','http://author/url-t') + self.story.addToList('authorUrl','http://author/url-y') + self.story.addToList('authorUrl','http://author/url-u') + self.story.addToList('authorUrl','http://author/url-i') + self.story.addToList('authorUrl','http://author/url-o') + + self.story.addToList('category','Power Rangers') + self.story.addToList('category','SG-1') + self.story.addToList('genre','Porn') + self.story.addToList('genre','Drama') + else: + self.story.setMetadata('authorId','98765') + self.story.setMetadata('authorUrl','http://author/url') + + self.story.addToList('warnings','Swearing') + self.story.addToList('warnings','Violence') + + if idstr == '80': + self.story.addToList('category',u'Rizzoli & Isles') + self.story.addToList('characters','J. Rizzoli') + elif idstr == '81': + self.story.addToList('category',u'Pitch Perfect') + self.story.addToList('characters','Chloe B.') + elif idstr == '82': + self.story.addToList('characters','Henry (Once Upon a Time)') + self.story.addToList('category',u'Once Upon a Time (TV)') + elif idstr == '83': + self.story.addToList('category',u'Rizzoli & Isles') + self.story.addToList('characters','J. Rizzoli') + self.story.addToList('category',u'Pitch Perfect') + self.story.addToList('characters','Chloe B.') + self.story.addToList('ships','Chloe B. & J. Rizzoli') + elif idstr == '90': + self.story.setMetadata('characters','Henry (Once Upon a Time)') + self.story.setMetadata('category',u'Once Upon a Time (TV)') + else: + self.story.addToList('category','Harry Potter') + self.story.addToList('category','Furbie') + self.story.addToList('category','Crossover') + self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ') + self.story.addToList('category',u'Magical Girl Lyrical Nanoha') + self.story.addToList('category',u'Once Upon a Time (TV)') + self.story.addToList('characters','Bob Smith') + self.story.addToList('characters','George Johnson') + self.story.addToList('characters','Fred Smythe') + self.story.addToList('ships','Harry Potter/Ginny Weasley') + self.story.addToList('ships','Harry Potter/Ginny Weasley/Albus Dumbledore') + self.story.addToList('ships','Harry Potter & Hermione Granger') + + self.story.addToList('genre','Fantasy') + self.story.addToList('genre','Comedy') + self.story.addToList('genre','Sci-Fi') + self.story.addToList('genre','Noir') + + self.story.addToList('listX','xVal1') + self.story.addToList('listX','xVal2') + self.story.addToList('listX','xVal3') + self.story.addToList('listX','xVal4') + + self.story.addToList('listY','yVal1') + self.story.addToList('listY','yVal2') + self.story.addToList('listY','yVal3') + self.story.addToList('listY','yVal4') + + self.story.addToList('listZ','zVal1') + self.story.addToList('listZ','zVal2') + self.story.addToList('listZ','zVal3') + self.story.addToList('listZ','zVal4') + + self.story.setMetadata('metaA','98765') + self.story.setMetadata('metaB','01245') + self.story.setMetadata('metaC','The mighty metaC!') + + self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"), + ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"), + ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"), + ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"), + ('Chapter 4',self.url+"&chapter=5"), + ('Chapter 5',self.url+"&chapter=6"), + ('Chapter 6',self.url+"&chapter=7"), + ('Chapter 7',self.url+"&chapter=8"), + ('Chapter 8',self.url+"&chapter=9"), + #('Chapter 9',self.url+"&chapter=0"), + #('Chapter 0',self.url+"&chapter=a"), + #('Chapter a',self.url+"&chapter=b"), + #('Chapter b',self.url+"&chapter=c"), + #('Chapter c',self.url+"&chapter=d"), + #('Chapter d',self.url+"&chapter=e"), + #('Chapter e',self.url+"&chapter=f"), + #('Chapter f',self.url+"&chapter=g"), + #('Chapter g',self.url+"&chapter=h"), + #('Chapter h',self.url+"&chapter=i"), + #('Chapter i',self.url+"&chapter=j"), + #('Chapter j',self.url+"&chapter=k"), + #('Chapter k',self.url+"&chapter=l"), + #('Chapter l',self.url+"&chapter=m"), + #('Chapter m',self.url+"&chapter=n"), + #('Chapter n',self.url+"&chapter=o"), + ] + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + + def getChapterText(self, url): + logger.debug('Getting chapter text from: %s' % url) + self.do_sleep() + if self.story.getMetadata('storyId').startswith('670') or \ + self.story.getMetadata('storyId').startswith('672'): + time.sleep(1.0) + + if "chapter=1" in url : + text=u''' +<div> +<h3>Prologue</h3> +<p>This is a fake adapter for testing purposes. Different sid's will give different errors:</p> +<h4>Config(personal.ini)</h4> +<p>sid>=1000 will use custom test story data from your configuration(personal.ini)</p> +<p>Hard coded ids:</p> +<p>http://test1.com?sid=664 - Crazy string title</p> +<p>http://test1.com?sid=665 - raises AdultCheckRequired</p> +<p>http://test1.com?sid=666 - raises StoryDoesNotExist</p> +<p>http://test1.com?sid=667 - raises FailedToDownload on chapters 2+</p> +<p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p> +<p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p> +<p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p> +<p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p> +<p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p> +<p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p> +<p>http://test1.com?sid=0 - Succeeds, generates some text specifically for testing hyphenation problems with Nook STR/STRwG</p> +<p>Odd sid's will be In-Progress, evens complete. sid<10 will be assigned one of four languages and included in a series.</p> +</div> +''' + elif self.story.getMetadata('storyId') == '0': + text=u'''<div> +<h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3> +<br /> + eyes… but I’m not convinced we should automatically<br /> +<br /><br /> +<b>Thanks to the latest to recommend me: Alastor</b><br /> +<br /><br /> + “Sure, invite her along. Does she have children?”<br /> +<br /> +</div> +''' + else: + if self.story.getMetadata('storyId') == '667': + raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url) + + text=u''' +<div> +<h3 extra="value">Chapter title from site</h3> +<p>Timestamp:'''+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+'''</p> +<p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p> +br breaks<br><br> +Puella Magi Madoka Magica/魔法少女まどか★マギカ +<!-- a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim & Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a --><br/> +br breaks<br><br> +Don't—e;ver—d;o—that—a;gain, 法 é +<hr> +horizontal rules +<hr size=1 noshade> +<p>"Lorem ipsum dolor sit amet", consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore--et dolore magna aliqua. 'Ut enim ad minim veniam', quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p> +<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p> +</div> +''' + soup = self.make_soup(text) + return self.utf8FromSoup(url,soup) + +def getClass(): + return TestSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_thealphagatecom.py b/fanficdownloader/adapters/adapter_thealphagatecom.py new file mode 100644 index 0000000..4420616 --- /dev/null +++ b/fanficdownloader/adapters/adapter_thealphagatecom.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return TheAlphaGateComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class TheAlphaGateComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','tag') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %b %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.thealphagate.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_thehexfilesnet.py b/fanficdownloader/adapters/adapter_thehexfilesnet.py new file mode 100644 index 0000000..1316b08 --- /dev/null +++ b/fanficdownloader/adapters/adapter_thehexfilesnet.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return TheHexFilesNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class TheHexFilesNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','thf') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y.%m.%d" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'thehexfiles.net' + + @classmethod + def getAcceptDomains(cls): + return ['www.thehexfiles.net','thehexfiles.net'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',stripHTML(a)) + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + + try: + # in case link points somewhere other than the first chapter + a = soup.findAll('option')[1]['value'] + self.story.setMetadata('storyId',a.split('=',)[1]) + url = 'http://'+self.host+'/'+a + soup = bs.BeautifulSoup(self._fetchUrl(url)) + except: + pass + + for info in asoup.findAll('table', {'cellspacing' : '4'}): + a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + if a != None: + self.story.setMetadata('title',stripHTML(a)) + break + + + # Find the chapters: + chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')) + if len(chapters) == 0: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + for chapter in chapters: + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + + cats = info.findAll('a',href=re.compile('categories.php')) + for cat in cats: + self.story.addToList('category',cat.string) + + words = info.find(text=re.compile('Words:')).split('|')[1].split(': ')[1] + self.story.setMetadata('numWords', words) + + comp = info.find('span', {'class' : 'completed'}).string.split(': ')[1] + if 'Yes' in comp: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + summary = info.find('td', {'class' : 'summary'}) + summary.name='div' # change td to div so it doesn't mess up the display when using table titlepage. + self.setDescription(url,summary) + + rating=stripHTML(info.find('td', {'align' : 'left'})).split('(')[1].split(')')[0] + self.story.setMetadata('rating', rating) + + labels = info.findAll('td', {'width' : '10%'}) + values = info.findAll('td', {'width' : '40%'}) + for i in range(0,len(labels)): + value = stripHTML(values[i]) + label = stripHTML(labels[i]) + + if 'Genres' in label: + genres = value.split(', ') + for genre in genres: + if genre != 'none': + self.story.addToList('genre',genre) + + + if 'Warnings' in label: + warnings = value.split(', ') + for warning in warnings: + if warning != 'none': + self.story.addToList('warnings',warning) + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr','img')) # otherwise soup eats the br/hr tags. + + if None == soup: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + # Ugh. chapter html doesn't haven't anything useful around it to demarcate. + for a in soup.findAll('table'): + a.extract() + + for a in soup.findAll('head'): + a.extract() + + html = soup.find('html') + html.name='div' + + return self.utf8FromSoup(url,soup) diff --git a/fanficdownloader/adapters/adapter_thehookupzonenet.py b/fanficdownloader/adapters/adapter_thehookupzonenet.py new file mode 100644 index 0000000..2b6c52e --- /dev/null +++ b/fanficdownloader/adapters/adapter_thehookupzonenet.py @@ -0,0 +1,309 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# By virtue of being recent and requiring both is_adult and user/pass, +# adapter_fanficcastletvnet.py is the best choice for learning to +# write adapters--especially for sites that use the eFiction system. +# Most sites that have ".../viewstory.php?sid=123" in the story URL +# are eFiction. + +# For non-eFiction sites, it can be considerably more complex, but +# this is still a good starting point. + +# In general an 'adapter' needs to do these five things: + +# - 'Register' correctly with the downloader +# - Site Login (if needed) +# - 'Are you adult?' check (if needed--some do one, some the other, some both) +# - Grab the chapter list +# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page) +# - Grab the chapter texts + +# Search for XXX comments--that's where things are most likely to need changing. + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return TheHookupZoneNetAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the /fanfic part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/CriminalMinds/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','thupz') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%b %d, %Y" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'thehookupzone.net' # XXX + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/CriminalMinds/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if "Age Consent Required" in data: # XXX + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/CriminalMinds/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/CriminalMinds/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + ## Not all sites use Genre, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + ## Not all sites use Warnings, but there's no harm to + ## leaving it in. Check to make sure the type_id number + ## is correct, though--it's site specific. + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/CriminalMinds/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_themaplebookshelf.py b/fanficdownloader/adapters/adapter_themaplebookshelf.py new file mode 100644 index 0000000..1d55deb --- /dev/null +++ b/fanficdownloader/adapters/adapter_themaplebookshelf.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +from base_efiction_adapter import BaseEfictionAdapter + +class TheMapleBookshelfComSiteAdapter(BaseEfictionAdapter): + + @staticmethod + def getSiteDomain(): + return 'themaplebookshelf.com' + + @classmethod + def getPathToArchive(self): + return '/Literati' + + @classmethod + def getSiteAbbrev(seluuf): + return 'maplebook' + + @classmethod + def getDateFormat(self): + return "%b %d, %Y" + +def getClass(): + return TheMapleBookshelfComSiteAdapter diff --git a/fanficdownloader/adapters/adapter_themasquenet.py b/fanficdownloader/adapters/adapter_themasquenet.py new file mode 100644 index 0000000..db65ee6 --- /dev/null +++ b/fanficdownloader/adapters/adapter_themasquenet.py @@ -0,0 +1,274 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return TheMasqueNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class TheMasqueNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + if self.parsedUrl.path.split('/',)[1] == 'wiktt': + self.story.addToList("category","Harry Potter") + self.section='/wiktt/efiction/' + self.dateformat = "%m/%d/%Y" + else: + self.story.addToList("category","Originals") + self.section='/efiction/' + self.dateformat = "%b %d, %Y" + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + self.section + 'viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','msq') + + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'themasque.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain())+"(/wiktt)?/efiction"+re.escape("/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + self.section + 'user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.section + chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + +# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.text + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_thepetulantpoetesscom.py b/fanficdownloader/adapters/adapter_thepetulantpoetesscom.py new file mode 100644 index 0000000..b35eaa5 --- /dev/null +++ b/fanficdownloader/adapters/adapter_thepetulantpoetesscom.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return ThePetulantPoetessComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class ThePetulantPoetessComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1') + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','tpp') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%Y/%m/%d" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.thepetulantpoetess.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'You must be a member to read this story.' in data \ + or "The Ministry of Magic does not have a record of that password. " in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "My Account Page" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + chapters=soup.find('select', {'name' : 'sid'}) + if chapters == None: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + for chapter in chapters.findAll('option', value=re.compile(r"viewstory.php\?sid=\d+&i=1")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['value'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # make sure that the story id is from the first chapter + self.story.setMetadata('storyId',self.chapterUrls[0][1].split('=')[1].split('&')[0]) + + #locate the story on author's page + index = 1 + found = 0 + while found == 0: + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&page="+str(index))) + + for info in asoup.findAll('td', {'class' : 'highlightcolor1'}): + a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + if a != None: + self.story.setMetadata('title',stripHTML(a)) + found = 1 + break + index=index+1 + + # extract metadata + b=info.find('b') + b.find('a').extract() + self.story.setMetadata('rating', b.text.split('[')[1].split(']')[0]) + + info = info.findNext('td', {'colspan' : '2'}) + for label in info.findAll('b'): + value = label.nextSibling + + if "Category" in label.text: + for cat in info.findAll('a'): + self.story.addToList('category',cat.string) + + if "Characters" in label.text: + for char in value.split(', '): + self.story.addToList('characters',char) + + if "Genres" in label.text: + for genre in value.split(', '): + if "General" not in genre: + self.story.addToList('genre',genre) + + if "Warnings" in label.text: + for warning in value.split(', '): + if "none" not in warning.lower(): + self.story.addToList('warnings',warning) + + + info = info.findNext('td', {'class' : 'tblborder'}) + info.find('b').extract() + self.setDescription(url,info) + + info = info.findNext('td', {'class' : 'highlightcolor2'}) + for label in info.findAll('b'): + value = label.nextSibling + + if "Completed" in label.text: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label.text: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label.text: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + if 'Word Count' in label.text: + self.story.setMetadata('numWords', value) + + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.findAll('table')[2].findAll('td')[1] + for a in div.findAll('div'): + a.extract() + div.name='div' + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_thequidditchpitchorg.py b/fanficdownloader/adapters/adapter_thequidditchpitchorg.py new file mode 100644 index 0000000..68401bf --- /dev/null +++ b/fanficdownloader/adapters/adapter_thequidditchpitchorg.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +# This function is called by the downloader in all adapter_*.py files +# in this dir to register the adapter class. So it needs to be +# updated to reflect the class below it. That, plus getSiteDomain() +# take care of 'Registering'. +def getClass(): + return TheQuidditchPitchOrgAdapter # XXX + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + # XXX Most sites don't have the part. Replace all to remove it usually. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','tqdpch') # XXX + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" # XXX + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'thequidditchpitch.org' # XXX + + @classmethod + def getAcceptDomains(cls): + return ['www.thequidditchpitch.org','thequidditchpitch.org'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only - Not suitable for readers under the age of legal consent in their country.' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" # XXX + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + if ("Not suitable for readers under the age of legal consent in their country." in data \ + or "Not suitable for readers under 16 yrs. \r\nStories may contain violence, slight nudity, and/or sexual situations." in data ) \ + and not (self.is_adult or self.getConfig("is_adult")): # XXX + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + #print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId'))) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + # span? Really? span? Yeah... I don't think so. + div = soup.find('span', {'style' : 'font-size: 100%;'}) + div.name='div' + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_tokrafandomnetcom.py b/fanficdownloader/adapters/adapter_tokrafandomnetcom.py new file mode 100644 index 0000000..1d3a927 --- /dev/null +++ b/fanficdownloader/adapters/adapter_tokrafandomnetcom.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return TokraFandomnetComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class TokraFandomnetComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','tokra') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. But it + # doesn't matter too much anymore. + return 'tokra.fandomnet.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=3" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + #print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Rating + rate = stripHTML(soup.find('div',{'id':'pagetitle'})) + rate = rate[rate.rindex('[')+1:rate.rindex(']')] + self.story.setMetadata('rating', rate) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + metadiv = soup.find('div',{'class':'content'}) + smalldiv = metadiv.find('div',{'class':'small'}) + + # tokra categories -> genre + # categories will be filled from ini. + genres = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for genre in genres: + self.story.addToList('genre',genre.string) + + chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + metatext = stripHTML(smalldiv) + + if 'Completed: Yes' in metatext: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + wordstart=metatext.rindex('Word count:')+12 + words = metatext[wordstart:metatext.index(' ',wordstart)] + self.story.setMetadata('numWords', words) + + datesdiv = soup.find('div',{'class':'bottom'}) + dates = stripHTML(datesdiv).split() + # Published: 04/26/2011 Updated: 03/06/2013 + self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # remove 'small' leaving only summary. + smalldiv.extract() + self.setDescription(url,metadiv) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url)) + + div = soup.find('div', {'class' : 'content'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + # remove some decorations while keeping notes. + remove = div.find('div', {'id' : 'pagetitle'}) + remove.extract() + + for remove in div.findAll('div', {'class' : 'right'}): + remove.extract() + + for remove in div.findAll('div', {'class' : 'left'}): + remove.extract() + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_tolkienfanfiction.py b/fanficdownloader/adapters/adapter_tolkienfanfiction.py new file mode 100644 index 0000000..b964baf --- /dev/null +++ b/fanficdownloader/adapters/adapter_tolkienfanfiction.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +""" +FFDL Adapter for TolkienFanFiction.com. + +Chapter URL: http://www.tolkienfanfiction.com/Story_Read_Chapter.php?CHid=1234 + Metadata + Link to Story URL [Index] + chapterTitle + storyTitle +Story URL: http://www.tolkienfanfiction.com/Story_Read_Head.php?STid=1034 + Metadata + Links to Chapter URLs + storyTitle + chapterTitle[s] + author + authorId + authorUrl + numChapters + wordCount + description/summary + rating TODO + genre TODO + Characters + Ages (specific) TODO +Search: http://www.tolkienfanfiction.com/Story_Chapter_Search.php?text=From+Wilderness+to+Cities+White&field=1&type=3&search=Search + Strategy + Search by exact phrase for styo + Metadata + dateUpdated + Parameters + field (field to search) + 1: title + 2: description + 3: chapter text + type (any, all or exact phrase) + 1: any + 2: all + 3: exact phrase + +""" +# Copyright 2014 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib +import urllib2 +import urlparse +import string + +from .. import BeautifulSoup as bs +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def _is_story_url(url): + return "Story_Read_Head.php" in url + +def _latinize(text): + """ + See http://stackoverflow.com/a/19114706/201318 + """ + src = u"áâäÉéêëíóôöúû" + tgt = u"aaaEeeeiooouu" + src_ord = [ord(char) for char in src] + translate_table = dict(zip(src_ord, tgt)) + return text.translate(translate_table) + +def _fix_broken_markup(html): + """Replaces invalid comment tags""" + if html.startswith("<CENTER>"): + logger.error("TolkienFanFiction.com couldn't handle this request: '%s'" % html) + html = re.sub("<!-.+?->", "", html) + return html + + +class TolkienFanfictionAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["ISO-8859-1", "Windows-1252"] + + self.story.setMetadata('siteabbrev','tolkien') + + self.dateformat = '%B %d, %Y' + + self._normalizeURL(url) + + def _normalizeURL(self, url): + if _is_story_url(url): + self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId')) + self._setURL('http://' + self.getSiteDomain() + '/Story_Read_Head.php?STid=' + self.story.getMetadata('storyId')) + + @staticmethod + def getSiteDomain(): + return 'tolkienfanfiction.com' + + @classmethod + def getAcceptDomains(cls): + return ['tolkienfanfiction.com', 'www.tolkienfanfiction.com'] + + @classmethod + def getSiteExampleURLs(cls): + return 'http://www.tolkienfanfiction.com/Story_Read_Head.php?STid=1034 http://www.tolkienfanfiction.com/Story_Read_Chapter.php?CHid=4945' + + def getSiteURLPattern(self): + return r"http://(?:www.)?tolkienfanfiction.com/(?:Story_Read_Chapter\.php\?CH|Story_Read_Head\.php\?ST)id=(?P<storyId>[0-9]+)" + + def extractChapterUrlsAndMetadata(self): + + if not _is_story_url(self.url): + # Get the link to the index page + try: + chapterHtml = _fix_broken_markup(self._fetchUrl(self.url)) + chapterSoup = bs.BeautifulSoup(chapterHtml) + indexLink = chapterSoup.find("a", text="[Index]").parent + self._normalizeURL('http://' + self.getSiteDomain() + '/' + indexLink.get('href')) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + logger.debug("Determined index page: <%s>" % self.url) + + try: + indexHtml = _fix_broken_markup(self._fetchUrl(self.url)) + soup = bs.BeautifulSoup(indexHtml) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # chapterUrls + for pfLink in soup.findAll("a", text='[PF] '): + chapterLink = pfLink.parent.findNext("a") + chapterTitle = chapterLink.string + if self.getConfig('strip_chapter_numeral'): + chapterTitle = re.sub("^\d+:", "", chapterTitle) + chapterUrl = 'http://' + self.host + '/' + chapterLink['href'] + self.chapterUrls.append((chapterTitle, chapterUrl)) + numChapters = len(self.chapterUrls) + self.story.setMetadata('numChapters', numChapters) + logger.debug('Number of Chapters: %s' % numChapters) + + # title + title = soup.find("table", "headertitle").find("tr").contents[1].string + logger.debug("Title: '%s'" % title) + self.story.setMetadata('title', title) + + # author + authorLink = soup.find("a", {"href":lambda x: x.startswith("Author_Profile.php")}) + authorName = authorLink.find("b").string + authorHref = authorLink['href'] + authorUrl = 'http:' + self.host + '/' + authorHref + authorId = authorHref[authorHref.index('=')+1:] + self.story.setMetadata('author', authorName) + self.story.setMetadata('authorId', authorId) + self.story.setMetadata('authorUrl', authorUrl) + logger.debug("Author: %s [%s] @ <%s>" % (authorId, authorName, authorUrl)) + + # numWords + numWordsMatch = re.search("Word Count: (\d+)<BR>", indexHtml) + if numWordsMatch: + numWords = numWordsMatch.group(1) + logger.debug('Number of words: %s' % numWords) + self.story.setMetadata('numWords', numWords) + + # description + description = soup.find("b", text="Description:").parent.nextSibling.nextSibling + self.story.setDescription(description) + logger.debug("Summary: '%s'" % description) + + # characters + characters = soup.find("b", text="Characters").parent.nextSibling.nextSibling.nextSibling + for character in characters.split(", "): + self.story.addToList('characters', character) + logger.debug("Characters: %s" % self.story.getMetadata('characters')) + + logger.debug('Title as `str`: ' + str(title)) + # For publication date we need to search + try: + queryString = urllib.urlencode(( + ('type', 3), + ('field', 1), + # need translate here for the weird accented letters + ('text', _latinize(title)), + ('search', 'Search'), + )) + searchUrl = 'http://%s/Story_Chapter_Search.php?%s' % (self.host, queryString) + logger.debug("Search URL: <%s>" % searchUrl) + searchHtml = _fix_broken_markup(self._fetchUrl(searchUrl)) + searchSoup = bs.BeautifulSoup(searchHtml) + date = searchSoup.find(text="Updated:").nextSibling.string + logger.debug("Last Updated: '%s'" % date) + self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat)) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + def getChapterText(self, url): + + logger.debug('Downloading chapter <%s>' % url) + + time.sleep(0.5) + htmldata = _fix_broken_markup(self._fetchUrl(url)) + soup = bs.BeautifulSoup(htmldata) + + #strip comments from soup + [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))] + + # Strip redundant headings + [font.parent.extract() for font in soup.findAll("font", {"size": "4"})] + + # get story text + textDiv = soup.find("div", "text") + return self.utf8FromSoup(url, textDiv) + +def getClass(): + return TolkienFanfictionAdapter diff --git a/fanficdownloader/adapters/adapter_trekiverseorg.py b/fanficdownloader/adapters/adapter_trekiverseorg.py new file mode 100644 index 0000000..7b44068 --- /dev/null +++ b/fanficdownloader/adapters/adapter_trekiverseorg.py @@ -0,0 +1,325 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return TrekiverseOrgAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class TrekiverseOrgAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["iso-8859-1", + "Windows-1252"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + # normalized story URL. + self._setURL("http://"+self.getSiteDomain()\ + +"/efiction/viewstory.php?sid="+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','trkvs') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d/%m/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. + return 'trekiverse.org' + + @classmethod + def getAcceptDomains(cls): + return ['trekiverse.org','efiction.trekiverse.org'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://efiction."+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return r'(http://trekiverse\.org/efiction/viewstory\.php\?sid=\d+|http://efiction\.trekiverse\.org/viewstory\.php\?sid=\d+)' + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/efiction/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&index=1&ageconsent=ok&warning=5" + else: + addurl="&index=1" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + # Now go hunting for all the meta data and the chapter list. + + ## Title and author + a = soup.find('div', {'id' : 'pagetitle'}) + aut = a.find('a', href=re.compile(r"^viewuser\.php\?uid=")) + self.story.setMetadata('authorId',aut['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/efiction/'+aut['href']) + self.story.setMetadata('author',aut.string) + + ttl = a.find('a', href=re.compile(r'^viewstory.php\?sid=%s$'%self.story.getMetadata('storyId'))) + self.story.setMetadata('title',ttl.string) + + # Find the chapters: + outputdiv = soup.find('div', {'id':'output'}) + # (amp;)? because it should be &, but is escaped to & in URL. + # viewstory.php?sid=35&chapter=3 + chapters=outputdiv.findAll('a', href=re.compile(r'^viewstory.php\?sid=%s&(amp;)?chapter=\d+$'%self.story.getMetadata('storyId'))) + if len(chapters)==0: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.") + if len(chapters)==1: + self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/efiction/'+chapters[0]['href'])) + else: + for chapter in chapters: + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = '' + while value and not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + # sometimes poorly formated desc (<p> w/o </p>) leads + # to all labels being included. + svalue=svalue[:svalue.find('<span class="label">')] + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=9')) + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + if 'Awards' in label: + awards = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=12')) + awardstext = [award.string for award in awards] + self.award = ', '.join(awardstext) + for award in awardstext: + self.story.addToList('awards',award.string) + + if 'Pairing' in label: + ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=10')) + shipstext = [ship.string for ship in ships] + self.ship = ', '.join(shipstext) + for ship in shipstext: + self.story.addToList('ships',ship.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=11')) + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y")) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y")) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/efiction/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + notesdiv = soup.find('div', {'class':'noteinfo'}) + if notesdiv != None: + div.insert(0,"<hr>") + div.insert(0,notesdiv) + div.insert(0,"<hr>") + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_tthfanficorg.py b/fanficdownloader/adapters/adapter_tthfanficorg.py new file mode 100644 index 0000000..84c06b2 --- /dev/null +++ b/fanficdownloader/adapters/adapter_tthfanficorg.py @@ -0,0 +1,310 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 +import time + +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','tth') + self.dateformat = u"%d\u00a0%b\u00a0%y" #   becomes \u00a0 with bs4/html5lib. + self.is_adult=False + self.username = None + self.password = None + # get storyId from url--url validation guarantees query correct + m = re.match(self.getSiteURLPattern(),url) + if m: + self.story.setMetadata('storyId',m.group('id')) + + # normalized story URL. + self._setURL("http://"+self.getSiteDomain()\ + +"/Story-"+self.story.getMetadata('storyId')) + else: + raise exceptions.InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + @staticmethod + def getSiteDomain(): + return 'www.tthfanfic.org' + + @classmethod + def getSiteExampleURLs(cls): + return "http://www.tthfanfic.org/Story-1234 http://www.tthfanfic.org/Story-1234/Author+Story+Title.htm http://www.tthfanfic.org/T-99999999/Story-1234-1/Author+Story+Title.htm http://www.tthfanfic.org/story.php?no=12345" + + # http://www.tthfanfic.org/T-999999999999/Story-12345-1/Author+Story+Title.htm + # http://www.tthfanfic.org/Story-12345 + # http://www.tthfanfic.org/Story-12345/Author+Story+Title.htm + # http://www.tthfanfic.org/story.php?no=12345 + def getSiteURLPattern(self): + return r"http://www.tthfanfic.org(/(T-\d+/)?Story-|/story.php\?no=)(?P<id>\d+)(-\d+)?(/.*)?$" + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return True + + # tth won't send you future updates if you aren't 'caught up' + # on the story. Login isn't required for F21, but logging in will + # mark stories you've downloaded as 'read' on tth. + def performLogin(self): + params = {} + + if self.password: + params['urealname'] = self.username + params['password'] = self.password + else: + params['urealname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['loginsubmit'] = 'Login' + + if not params['password']: + return + + loginUrl = 'http://' + self.getSiteDomain() + '/login.php' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['urealname'])) + + ## need to pull empty login page first to get ctkn and + ## password name, which are BUSs +# <form method='post' action='/login.php' accept-charset="utf-8"> +# <input type='hidden' name='ctkn' value='4bdf761f5bea06bf4477072afcbd0f8d721d1a4f989c09945a9e87afb7a66de1'/> +# <input type='text' id='urealname' name='urealname' value=''/> +# <input type='password' id='password' name='6bb3fcd148d148629223690bf19733b8'/> +# <input type='submit' value='Login' name='loginsubmit'/> + soup = self.make_soup(self._fetchUrl(loginUrl)) + params['ctkn']=soup.find('input', {'name':'ctkn'})['value'] + params[soup.find('input', {'id':'password'})['name']] = params['password'] + + d = self._fetchUrl(loginUrl, params) + + if "Stories Published" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['urealname'])) + raise exceptions.FailedToLogin(self.url,params['urealname']) + return False + else: + return True + + def extractChapterUrlsAndMetadata(self): + # fetch the chapter. From that we will get almost all the + # metadata and chapter list + + url=self.url + logger.debug("URL: "+url) + + # tth won't send you future updates if you aren't 'caught up' + # on the story. Login isn't required for F21, but logging in will + # mark stories you've downloaded as 'read' on tth. + self.performLogin() + + # use BeautifulSoup HTML parser to make everything easier to find. + try: + data = self._fetchUrl(url) + #print("data:%s"%data) + soup = self.make_soup(data) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(url) + else: + raise e + + descurl = url + + if "<h2>Story Not Found</h2>" in data: + raise exceptions.StoryDoesNotExist(url) + + if self.is_adult or self.getConfig("is_adult"): + form = soup.find('form', {'id':'sitemaxratingform'}) + # if is_adult and rating isn't already set to FR21, set it so. + if not form.find('option',{'value':'5'}).get('selected'): + params={'ctkn':form.find('input', {'name':'ctkn'})['value'], + 'sitemaxrating':'5'} + logger.info("Attempting to get rating cookie for %s" % url) + data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params) + # refetch story page. + ## XXX - needs cache invalidate? Or at least check that it this needs doing... + data = self._fetchUrl(url,usecache=False) + soup = self.make_soup(data) + + if "NOTE: This story is rated FR21 which is above your chosen filter level." in data: + raise exceptions.AdultCheckRequired(self.url) + + # http://www.tthfanfic.org/AuthorStories-3449/Greywizard.htm + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"^/AuthorStories-\d+")) + self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+a['href']) + self.story.setMetadata('author',stripHTML(a)) + authorurl = 'http://'+self.host+a['href'] + + try: + # going to pull part of the meta data from *primary* author list page. + logger.debug("**AUTHOR** URL: "+authorurl) + authordata = self._fetchUrl(authorurl) + descurl=authorurl + authorsoup = self.make_soup(authordata) + # author can have several pages, scan until we find it. + while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ): + nextarrow = authorsoup.find('a', {'class':'arrowf'}) + if not nextarrow: + ## if rating is set lower than story, it won't be + ## visible on author lists unless. The *story* is + ## visible via the url, just not the entry on + ## author list. + raise exceptions.AdultCheckRequired(self.url) + nextpage = 'http://'+self.host+nextarrow['href'] + logger.debug("**AUTHOR** nextpage URL: "+nextpage) + authordata = self._fetchUrl(nextpage) + descurl=nextpage + authorsoup = self.make_soup(authordata) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(url) + else: + raise e + + storydiv = authorsoup.find('div', {'id':'st'+self.story.getMetadata('storyId'), 'class':re.compile(r"storylistitem")}) + self.setDescription(descurl,storydiv.find('div',{'class':'storydesc'})) + #self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'}))) + self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'}))) + + ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId')) + if ainfo != None: # indicates multiple authors/contributors. + try: + # going to pull part of the meta data from author list page. + infourl = 'http://'+self.host+ainfo['href'] + logger.debug("**StoryInfo** URL: "+infourl) + infodata = self._fetchUrl(infourl) + infosoup = self.make_soup(infodata) + + # for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")): + # self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1]) + # self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-")) + # self.story.addToList('author',stripHTML(a)) + + # second verticaltable is the chapter list. + table = infosoup.findAll('table',{'class':'verticaltable'})[1] + for a in table.findAll('a',href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))): + autha = a.findNext('a',href=re.compile(r"^/Author-\d+")) + self.story.addToList('authorId',autha['href'].split('/')[1].split('-')[1]) + self.story.addToList('authorUrl','http://'+self.host+autha['href'].replace("/Author-","/AuthorStories-")) + self.story.addToList('author',stripHTML(autha)) + # include leading number to match 1. ... 2. ... + self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1, + stripHTML(a), + stripHTML(autha)),'http://'+self.host+a['href'])) + + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(url) + else: + raise e + else: # single author: + # Find the chapter selector + select = soup.find('select', { 'name' : 'chapnav' } ) + + if select is None: + # no selector found, so it's a one-chapter story. + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + allOptions = select.findAll('option') + for o in allOptions: + url = "http://"+self.host+o['value'] + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(o),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + verticaltable = soup.find('table', {'class':'verticaltable'}) + + BtVS = True + BtVSNonX = False + for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")): + if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'Non-BTVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']: + self.story.addToList('category',cat.string) + else: + if 'Non-BtVS' in cat.string or 'Non-BTVS' in cat.string: + BtVS = False + if 'BtVS/AtS Non-Crossover' == cat.string: + BtVSNonX = True + + verticaltabletds = verticaltable.findAll('td') + self.story.setMetadata('rating', verticaltabletds[2].string) + self.story.setMetadata('numWords', verticaltabletds[4].string) + + # Complete--if completed. + if 'Yes' in verticaltabletds[10].string: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + #print("date:%s"%verticaltabletds[8]) + self.story.setMetadata('datePublished',makeDate(stripHTML(verticaltabletds[8].string), self.dateformat)) + self.story.setMetadata('dateUpdated',makeDate(stripHTML(verticaltabletds[9].string), self.dateformat)) + + for icon in storydiv.find('span',{'class':'storyicons'}).findAll('img'): + if( icon['title'] not in ['Non-Crossover'] ) : + self.story.addToList('genre',icon['title']) + else: + if not BtVSNonX: + BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover + + #print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX)) + if BtVS: + self.story.addToList('category','Buffy: The Vampire Slayer') + + pseries = soup.find('p', {'style':'margin-top:0px'}) + #print("pseries:%s"%pseries.get_text()) + m = re.match('This story is No\. (?P<num>\d+) in the series "(?P<series>.+)"\.', + pseries.get_text()) + if m: + self.setSeries(m.group('series'),m.group('num')) + self.story.setMetadata('seriesUrl',"http://"+self.host+pseries.find('a')['href']) + + def getChapterText(self, url): + logger.debug('Getting chapter text from: %s' % url) + soup = self.make_soup(self._fetchUrl(url)) + + div = soup.find('div', {'id' : 'storyinnerbody'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + # strip out included chapter title, if present, to avoid doubling up. + try: + div.find('h3').extract() + except: + pass + return self.utf8FromSoup(url,div) + +def getClass(): + return TwistingTheHellmouthSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_twcslibrarynet.py b/fanficdownloader/adapters/adapter_twcslibrarynet.py new file mode 100644 index 0000000..6bae06a --- /dev/null +++ b/fanficdownloader/adapters/adapter_twcslibrarynet.py @@ -0,0 +1,272 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','twcs') + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + self.dateformat = "%d %b %Y" + + + @classmethod + def getAcceptDomains(cls): + return ['thewriterscoffeeshop.com','twcslibrary.net'] + + @staticmethod + def getSiteDomain(): + return 'www.twcslibrary.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+"(www.)?("+'|'.join(self.getAcceptDomains())+")/(library/)?"+re.escape("viewstory.php?sid=")+r"\d+$" + + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + addurl = "&ageconsent=ok&warning=3" + else: + addurl="" + + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + if "Age Consent Required" in data: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # problems with some stories, but only in calibre. I suspect + # issues with different SGML parsers in python. This is a + # nasty hack, but it works. + data = data[data.index("<body"):] + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + # poor HTML(unclosed <p> for one) can cause run on + # over the next label. + if '<span class="label">' in svalue: + svalue = svalue[0:svalue.find('<span class="label">')] + break + else: + value = value.nextSibling + self.setDescription(url,svalue) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class')) + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + # problems with some stories, but only in calibre. I suspect + # issues with different SGML parsers in python. This is a + # nasty hack, but it works. + data = data[data.index("<body"):] + + chapter=bs.BeautifulSoup('<div class="story"></div>') + + soup = bs.BeautifulSoup(data) + + found=False + for div in soup.findAll('div'): + if div.has_key('class') and div['class'] == 'notes': + chapter.append(div) + if div.has_key('id') and div['id'] == 'story': + chapter.append(div) + found=True + + if not found: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,chapter) + +def getClass(): + return TheWritersCoffeeShopComSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_twilightarchivescom.py b/fanficdownloader/adapters/adapter_twilightarchivescom.py new file mode 100644 index 0000000..2b8fc20 --- /dev/null +++ b/fanficdownloader/adapters/adapter_twilightarchivescom.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return TwilightArchivesComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class TwilightArchivesComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2]) + + + # normalized story URL. http://www.twilightarchives.com/read/9353 + self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','twa') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %b %y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.twilightarchives.com' + + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/read/1234" + + def getSiteURLPattern(self): + return re.escape("http://" + self.getSiteDomain()+"/read/")+r"\d+(/d+)?$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('h1') + self.story.setMetadata('title',stripHTML(a)) + + # Find the chapters: + chapters=soup.find('ol', {'class' : 'chapters'}) + if chapters != None: + for chapter in chapters.findAll('a', href=re.compile(r'/read/'+self.story.getMetadata('storyId')+"/\d+$")): + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href'])) + else: + self.chapterUrls.append((self.story.getMetadata('title'),url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + +# rated, genre, warnings, seires + + summary = soup.find('p', {'class' : 'images'}) + self.setDescription(url,summary) + + for c in soup.findAll('h2', {'class' : 'title'}): + div = c.nextSibling.nextSibling + + if 'Information' in c.text: + for dt in div.findAll('dt'): + dd=dt.nextSibling.nextSibling + + if 'Author' in dt.text: + a=dd.find('a') + self.story.setMetadata('authorId',a['href'].split('/')[2]) + self.story.setMetadata('authorUrl','http://'+self.host+a['href']) + self.story.setMetadata('author',a.text) + + if 'Words' in dt.text: + self.story.setMetadata('numWords', dd.text) + + if 'Published' in dt.text: + self.story.setMetadata('datePublished', makeDate(stripHTML(dd.text), self.dateformat)) + + if 'Updated' in dt.text: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(dd.text), self.dateformat)) + + if 'Status' in dt.text: + if 'Complete' in dd.text: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Categories' in c.text: + for a in div.findAll('a'): + self.story.addToList('category',a.text) + + if 'Characters' in c.text: + for a in div.findAll('a'): + self.story.addToList('category',a.text) + + if 'Series' in c.text: + a=div.find('a') + series_name = a.text + series_url = 'http://'+self.host+a['href'] + + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.find('tbody').findAll('a', href=re.compile(r'^/read/\d+$')) + i=1 + for a in storyas: + if a['href'] == ('/read/'+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) + a=asoup.find('tbody').find('a', href=re.compile(r'^/read/'+self.story.getMetadata('storyId'))) + self.story.setMetadata('rating',a.parent.nextSibling.nextSibling.text) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'class' : 'size images medium'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_twilightednet.py b/fanficdownloader/adapters/adapter_twilightednet.py new file mode 100644 index 0000000..e70932d --- /dev/null +++ b/fanficdownloader/adapters/adapter_twilightednet.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class TwilightedNetSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','tw') + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + + @staticmethod + def getSiteDomain(): + return 'www.twilighted.net' + + @classmethod + def getAcceptDomains(cls): + return ['www.twilighted.net','twilighted.net'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://www.twilighted.net/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+r"(www\.)?"+re.escape("twilighted.net/viewstory.php?sid=")+r"\d+$" + + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + def extractChapterUrlsAndMetadata(self): + + url = self.url+'&index=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # problems with some stories, but only in calibre. I suspect + # issues with different SGML parsers in python. This is a + # nasty hack, but it works. + # twilighted isn't writing <body> ??? wtf? + data = "<html><body>"+data[data.index("</head>"):] + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + ## twilighted.net doesn't use genre. + # if 'Genre' in label: + # genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class')) + # genrestext = [genre.string for genre in genres] + # self.genre = ', '.join(genrestext) + # for genre in genrestext: + # self.story.addToList('genre',genre.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(value.strip(), "%B %d, %Y")) + + if 'Updated' in label: + # there's a stray [ at the end. + #value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y")) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + # problems with some stories, but only in calibre. I suspect + # issues with different SGML parsers in python. This is a + # nasty hack, but it works. + # twilighted isn't writing <body> ??? wtf? + data = "<html><body>"+data[data.index("</head>"):] + + soup = bs.BeautifulStoneSoup(data, + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + span = soup.find('div', {'id' : 'story'}) + + if None == span: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,span) + +def getClass(): + return TwilightedNetSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_twiwritenet.py b/fanficdownloader/adapters/adapter_twiwritenet.py new file mode 100644 index 0000000..2e9ae98 --- /dev/null +++ b/fanficdownloader/adapters/adapter_twiwritenet.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class TwiwriteNetSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','twrt') + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.is_adult = False + self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all. + self.password = "" + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + + @staticmethod + def getSiteDomain(): + return 'www.twiwrite.net' + + @classmethod + def getAcceptDomains(cls): + return ['www.twiwrite.net','twiwrite.net'] + + @classmethod + def getSiteExampleURLs(cls): + return "http://www.twiwrite.net/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://")+r"(www\.)?"+re.escape("twiwrite.net/viewstory.php?sid=")+r"\d+$" + + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.info("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=1" # XXX + else: + addurl="" + + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + if "Contains Explicit Content for mature adults only! May contain graphic violence, mature sexual situations, and explicit language. Read with caution." in data: + raise exceptions.AdultCheckRequired(self.url) + + # problems with some stories, but only in calibre. I suspect + # issues with different SGML parsers in python. This is a + # nasty hack, but it works. + data = data[data.index("<body"):] + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + + pagetitlediv = soup.find('div',id='pagetitle') + + ## Title + a = pagetitlediv.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = pagetitlediv.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + ## <meta name='description' content='<p>Description</p> ...' > + ## Summary, strangely, is in the content attr of a <meta name='description'> tag + ## which is escaped HTML. Unfortunately, we can't use it because they don't + ## escape (') chars in the desc, breakin the tag. + #meta_desc = soup.find('meta',{'name':'description'}) + #metasoup = bs.BeautifulStoneSoup(meta_desc['content']) + #self.story.setMetadata('description',stripHTML(metasoup)) + + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) + genrestext = [genre.string for genre in genres] + self.genre = ', '.join(genrestext) + for genre in genrestext: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=8')) + warningstext = [warning.string for warning in warnings] + self.warning = ', '.join(warningstext) + for warning in warningstext: + self.story.addToList('warning',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(value.strip(), "%B %d, %Y")) + + if 'Updated' in label: + # there's a stray [ at the end. + value = value[0:-1] + self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y")) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + data = self._fetchUrl(url) + # problems with some stories, but only in calibre. I suspect + # issues with different SGML parsers in python. This is a + # nasty hack, but it works. + data = data[data.index("<body"):] + + soup = bs.BeautifulStoneSoup(data, + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + span = soup.find('div', {'id' : 'story'}) + + if None == span: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,span) + +def getClass(): + return TwiwriteNetSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_voracity2eficcom.py b/fanficdownloader/adapters/adapter_voracity2eficcom.py new file mode 100644 index 0000000..56c4257 --- /dev/null +++ b/fanficdownloader/adapters/adapter_voracity2eficcom.py @@ -0,0 +1,233 @@ +# Software: eFiction +import re +import urllib2 +import urlparse + +from .. import BeautifulSoup + +from base_adapter import BaseSiteAdapter, makeDate +from .. import exceptions + + +def getClass(): + return Voracity2EficComAdapter + + +# yields Tag _and_ NavigableString siblings from the given tag. The +# BeautifulSoup findNextSiblings() method for some reasons only returns either +# NavigableStrings _or_ Tag objects, not both. +def _yield_next_siblings(tag): + sibling = tag.nextSibling + while sibling: + yield sibling + sibling = sibling.nextSibling + + +class Voracity2EficComAdapter(BaseSiteAdapter): + SITE_ABBREVIATION = 'voe' + SITE_DOMAIN = 'voracity2.e-fic.com' + + BASE_URL = 'http://' + SITE_DOMAIN + '/' + LOGIN_URL = BASE_URL + 'user.php?action=login' + VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d' + METADATA_URL_SUFFIX = '&index=1' + AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=4' + + DATETIME_FORMAT = '%m/%d/%Y' + REQUIRED_SKIN = 'Simple Elegance' + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + query_data = urlparse.parse_qs(self.parsedUrl.query) + story_id = query_data['sid'][0] + + self.story.setMetadata('storyId', story_id) + self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id)) + self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION) + + self.is_logged_in = False + + def _login(self): + # Apparently self.password is only set when login fails, i.e. + # the FailedToLogin exception is raised, so the adapter gets new + # login data and tries again + if self.password: + password = self.password + username = self.username + else: + username = self.getConfig('username') + password = self.getConfig('password') + + parameters = { + 'penname': username, + 'password': password, + 'submit': 'Submit'} + + class CustomizedFailedToLogin(exceptions.FailedToLogin): + def __init__(self, url, passwdonly=False): + # Use username variable from outer scope + exceptions.FailedToLogin.__init__(self, url, username, passwdonly) + + soup = self._customized_fetch_url(self.LOGIN_URL, CustomizedFailedToLogin, parameters) + div = soup.find('div', id='useropts') + if not div: + raise CustomizedFailedToLogin(self.LOGIN_URL) + + self.is_logged_in = True + + def _customized_fetch_url(self, url, exception=None, parameters=None): + if exception: + try: + data = self._fetchUrl(url, parameters) + except urllib2.HTTPError: + raise exception(self.url) + # Just let self._fetchUrl throw the exception, don't catch and + # customize it. + else: + data = self._fetchUrl(url, parameters) + + return BeautifulSoup.BeautifulSoup(data) + + @staticmethod + def getSiteDomain(): + return Voracity2EficComAdapter.SITE_DOMAIN + + @classmethod + def getSiteExampleURLs(cls): + return cls.VIEW_STORY_URL_TEMPLATE % 1234 + + def getSiteURLPattern(self): + return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$' + + def extractChapterUrlsAndMetadata(self): + soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX) + + # Check if the story is for "Registered Users Only", i.e. has adult + # content. Based on the "is_adult" attributes either login or raise an + # error. + errortext_div = soup.find('div', {'class': 'errortext'}) + if errortext_div: + error_text = ''.join(errortext_div(text=True)).strip() + if error_text == 'Registered Users Only': + if not (self.is_adult or self.getConfig('is_adult')): + raise exceptions.AdultCheckRequired(self.url) + self._login() + else: + # This case usually occurs when the story doesn't exist, but + # might potentially be something else, so just raise + # FailedToDownload exception with the found error text. + raise exceptions.FailedToDownload(error_text) + + url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX]) + soup = self._customized_fetch_url(url) + + # If logged in and the skin doesn't match the required skin throw an + # error + if self.is_logged_in: + skin = soup.find('select', {'name': 'skin'}).find('option', selected=True)['value'] + if skin != self.REQUIRED_SKIN: + raise exceptions.FailedToDownload('Required skin "%s" must be set in preferences' % self.REQUIRED_SKIN) + + pagetitle_div = soup.find('div', id='pagetitle') + self.story.setMetadata('title', pagetitle_div.a.string) + + author_anchor = pagetitle_div.a.findNextSibling('a') + url = urlparse.urljoin(self.BASE_URL, author_anchor['href']) + components = urlparse.urlparse(url) + query_data = urlparse.parse_qs(components.query) + + self.story.setMetadata('author', author_anchor.string) + self.story.setMetadata('authorId', query_data['uid']) + self.story.setMetadata('authorUrl', url) + + sort_div = soup.find('div', id='sort') + self.story.setMetadata('reviews', sort_div('a')[1].string) + + for b_tag in soup.find('div', {'class': 'listbox'})('b'): + key = b_tag.string.strip(' :') + try: + value = b_tag.nextSibling.string.strip() + # This can happen with some fancy markup in the summary. Just + # ignore this error and set value to None, the summary parsing + # takes care of this + except AttributeError: + value = None + + if key == 'Summary': + contents = [] + keep_summary_html = self.getConfig('keep_summary_html') + + for sibling in _yield_next_siblings(b_tag): + if isinstance(sibling, BeautifulSoup.Tag): + # Encountered next label, break. This method is the + # safest and most reliable I could think of. Blame + # e-fiction sites that allow their users to include + # arbitrary markup into their summaries and the + # horrible HTML markup. + if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br': + break + + if keep_summary_html: + contents.append(self.utf8FromSoup(self.url, sibling)) + else: + contents.append(''.join(sibling(text=True))) + else: + contents.append(sibling) + + # Remove the preceding break line tag and other crud + contents.pop() + contents.pop() + self.story.setMetadata('description', ''.join(contents)) + + elif key == 'Rating': + self.story.setMetadata('rating', value) + + elif key == 'Category': + for sibling in b_tag.findNextSiblings(['a', 'br']): + if sibling.name == 'br': + break + self.story.addToList('category', sibling.string) + + # Seems to be always "None" for some reason + elif key == 'Characters': + for sibling in b_tag.findNextSiblings(['a', 'br']): + if sibling.name == 'br': + break + self.story.addToList('characters', sibling.string) + + elif key == 'Series': + a = b_tag.findNextSibling('a') + if not a: + continue + self.story.setMetadata('series', a.string) + self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href'])) + + elif key == 'Chapter': + self.story.setMetadata('numChapters', int(value)) + + elif key == 'Completed': + self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress') + + elif key == 'Words': + self.story.setMetadata('numWords', value) + + elif key == 'Read': + self.story.setMetadata('readings', value) + + elif key == 'Published': + self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT)) + + elif key == 'Updated': + self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT)) + + for b_tag in soup.find('div', id='output').findNextSiblings('b'): + chapter_anchor = b_tag.a + title = chapter_anchor.string + url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href']) + self.chapterUrls.append((title, url)) + + def getChapterText(self, url): + url += self.AGE_CONSENT_URL_SUFFIX + soup = self._customized_fetch_url(url) + return self.utf8FromSoup(url, soup.find('div', id='story')) diff --git a/fanficdownloader/adapters/adapter_walkingtheplankorg.py b/fanficdownloader/adapters/adapter_walkingtheplankorg.py new file mode 100644 index 0000000..8e0cb19 --- /dev/null +++ b/fanficdownloader/adapters/adapter_walkingtheplankorg.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return WalkingThePlankOrgAdapter + +class WalkingThePlankOrgAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','wtp') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%b %d, %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.walkingtheplank.org' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$" + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + + # The actual text that is used to announce you need to be an + # adult varies from site to site. Again, print data before + # the title search to troubleshoot. + + if "By clicking this link, you acknowledge" in data: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href']) + self.story.setMetadata('author',a.string) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + labels = soup.findAll('span',{'class':'label'}) + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + #self.story.setMetadata('description',stripHTML(svalue)) + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + catstext = [cat.string for cat in cats] + for cat in catstext: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + charstext = [char.string for char in chars] + for char in charstext: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/archive/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_whoficcom.py b/fanficdownloader/adapters/adapter_whoficcom.py new file mode 100644 index 0000000..099dcf4 --- /dev/null +++ b/fanficdownloader/adapters/adapter_whoficcom.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +class WhoficComSiteAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev','whof') + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + + @staticmethod + def getSiteDomain(): + return 'www.whofic.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+"\d+$" + + def extractChapterUrlsAndMetadata(self): + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + # fetch the first chapter. From that we will: + # - determine title, authorname, authorid + # - get chapter list, if not one-shot. + + url = self.url+'&chapter=1' + logger.debug("URL: "+url) + + # use BeautifulSoup HTML parser to make everything easier to find. + try: + soup = bs.BeautifulSoup(self._fetchUrl(url)) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # pull title(title) and author from the HTML title. + title = stripHTML(soup.find('title')) + logger.debug('Title: %s' % title) + title = title.split('::')[1].strip() + self.story.setMetadata('title',title.split(' by ')[0].strip()) + self.story.setMetadata('author',title.split(' by ')[1].strip()) + + # Find authorid and URL from... author url. + a = soup.find('a', href=re.compile(r"viewuser.php")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + + # Find the chapter selector + select = soup.find('select', { 'name' : 'chapter' } ) + + if select is None: + # no selector found, so it's a one-chapter story. + self.chapterUrls.append((self.story.getMetadata('title'),url)) + else: + allOptions = select.findAll('option') + for o in allOptions: + url = self.url + "&chapter=%s" % o['value'] + # just in case there's tags, like <i> in chapter titles. + title = "%s" % o + title = re.sub(r'<[^>]+>','',title) + self.chapterUrls.append((title,url)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + ## Whofic.com puts none of the other meta data in the chapters + ## or even the story chapter index page. Need to scrape the + ## author page to find it. + + # <table width="100%" bordercolor="#333399" border="0" cellspacing="0" cellpadding="2"><tr><td> + # <b><a href="viewstory.php?sid=38220">Accompaniment 2</a></b> by <a href="viewuser.php?uid=12412">clandestinemiscreant</a> [<a href="reviews.php?sid=38220">Reviews</a> - <a href="reviews.php?sid=38220">0</a>] <br> + # This is a series of short stories written as an accompaniment to Season 2, Season 28 for us oldies, and each is unrelated except for that one factor. Each story is canon, in that it does not change established events at time of airing, based on things mentioned and/or implied and missing or deleted scenes that were not seen in the final aired episodes.<br> + # <font size="-1"><b><a href="categories.php?catid=15">Tenth Doctor</a></b> - All Ages - None - Humor, Hurt/Comfort, Romance<br> + # <i>Characters:</i> Rose Tyler<br> + # <i>Series:</i> None<br> + # <i>Published:</i> 2010.08.15 - <i>Updated:</i> 2010.08.16 - <i>Chapters:</i> 4 - <i>Completed:</i> Yes - <i>Word Count:</i> 4890 </font> + # </td></tr></table> + + logger.debug("Author URL: "+self.story.getMetadata('authorUrl')) + soup = bs.BeautifulStoneSoup(self._fetchUrl(self.story.getMetadata('authorUrl')), + selfClosingTags=('br')) # normalize <br> tags to <br /> + # find this story in the list, parse it's metadata based on + # lots of assumptions about the html, since there's little + # tagging. + # Found a story once that had the story URL in the desc for a + # series on the same author's page. Now using the reviews + # link instead to find the appropriate metadata. + a = soup.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId'))) + metadata = a.findParent('td') + metadatachunks = self.utf8FromSoup(None,metadata).split('<br />') + # process metadata for this story. + self.setDescription(url,metadatachunks[1]) + #self.story.setMetadata('description', metadatachunks[1]) + + # First line of the stuff with ' - ' separators + moremeta = metadatachunks[2] + moremeta = re.sub(r'<[^>]+>','',moremeta) # strip tags. + + moremetaparts = moremeta.split(' - ') + + # first part is category--whofic.com has categories + # Doctor One-11, Torchwood, etc. We're going to + # prepend any with 'Doctor' or 'Era' (Multi-Era, Other + # Era) as 'Doctor Who'. + # + # Also push each in as 'extra tags'. + category = moremetaparts[0] + if 'Doctor' in category or 'Era' in category : + self.story.addToList('category','Doctor Who') + + for cat in category.split(', '): + self.story.addToList('category',cat) + + # next in that line is age rating. + self.story.setMetadata('rating',moremetaparts[1]) + + # after that is a possible list fo specific warnings, + # Explicit Violence, Swearing, etc + if "None" not in moremetaparts[2]: + for warn in moremetaparts[2].split(', '): + self.story.addToList('warnings',warn) + + # then genre. It's another comma list. All together + # in genre, plus each in extra tags. + genre=moremetaparts[3] + for g in genre.split(r', '): + self.story.addToList('genre',g) + + # line 3 is characters. + chars = metadatachunks[3] + charsearch="<i>Characters:</i>" + if charsearch in chars: + chars = chars[metadatachunks[3].index(charsearch)+len(charsearch):] + for c in chars.split(','): + if c.strip() != u'None': + self.story.addToList('characters',c) + + # the next line is stuff with ' - ' separators *and* names--with tags. + moremeta = metadatachunks[5] + moremeta = re.sub(r'<[^>]+>','',moremeta) # strip tags. + + moremetaparts = moremeta.split(' - ') + + for part in moremetaparts: + (name,value) = part.split(': ') + name=name.strip() + value=value.strip() + if name == 'Published': + self.story.setMetadata('datePublished', makeDate(value, '%Y.%m.%d')) + if name == 'Updated': + self.story.setMetadata('dateUpdated', makeDate(value, '%Y.%m.%d')) + if name == 'Completed': + if value == 'Yes': + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + if name == 'Word Count': + self.story.setMetadata('numWords', value) + + try: + # Find Series name from series URL. + a = metadata.find('a', href=re.compile(r"series.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + + # hardly a great identifier, I know, but whofic really doesn't + # give us anything better to work with. + span = soup.find('span', {'style' : 'font-size: 100%;'}) + + if None == span: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + span.name='div' + return self.utf8FromSoup(url,span) + +def getClass(): + return WhoficComSiteAdapter + diff --git a/fanficdownloader/adapters/adapter_wizardtalesnet.py b/fanficdownloader/adapters/adapter_wizardtalesnet.py new file mode 100644 index 0000000..3845196 --- /dev/null +++ b/fanficdownloader/adapters/adapter_wizardtalesnet.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return WizardTalesNetAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class WizardTalesNetAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','wzt') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.wizardtales.net' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Login seems to be reasonably standard across eFiction sites. + def needToLoginCheck(self, data): + if 'Registered Users Only' in data \ + or 'There is no such account on our website' in data \ + or "That password doesn't match the one in our database" in data: + return True + else: + return False + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login' + logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl, + params['penname'])) + + d = self._fetchUrl(loginUrl, params) + + if "Member Account" not in d : #Member Account + logger.info("Failed to login to URL %s as %s" % (loginUrl, + params['penname'])) + raise exceptions.FailedToLogin(url,params['penname']) + return False + else: + return True + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=4" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if self.needToLoginCheck(data): + # need to log in for this one. + self.performLogin(url) + data = self._fetchUrl(url) + + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logger.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + pt = soup.find('div', {'id' : 'pagetitle'}) + a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) + self.story.setMetadata('author',a.string) + + rating=pt.text.split('[')[1].split(']')[0] + self.story.setMetadata('rating', rating) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # <span class="label">Rated:</span> NC-17<br /> etc + content=soup.find('div',{'class' : 'content'}) + + for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')): + self.story.addToList('genre',genre.string) + + for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')): + self.story.addToList('warnings',warning.string) + + labels = content.findAll('b') + + value = labels[0].previousSibling + svalue = "" + while value != None: + val = value + value = value.previousSibling + while "Categories" not in val: + svalue += str(val) + val = val.nextSibling + self.setDescription(url,svalue) + + + + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Summary' in label: + ## Everything until the next span class='label' + svalue = "" + while not defaultGetattr(value,'class') == 'label': + svalue += str(value) + value = value.nextSibling + self.setDescription(url,svalue) + + if 'Word count' in label: + self.story.setMetadata('numWords', stripHTML(value).split(': ')[1].split(';')[0]) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Completed' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(stripHTML(value).split(': ')[1].split(';')[0], self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value).split(': ')[1], self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.findAll('div', {'id' : 'story'})[1] + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_wolverineandroguecom.py b/fanficdownloader/adapters/adapter_wolverineandroguecom.py new file mode 100644 index 0000000..d2d88a3 --- /dev/null +++ b/fanficdownloader/adapters/adapter_wolverineandroguecom.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +def getClass(): + return WolverineAndRogueComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class WolverineAndRogueComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + # normalized story URL. + self._setURL('http://' + self.getSiteDomain() + '/wrfa/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','wrfa') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%m/%d/%Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.wolverineandrogue.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/wrfa/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=")+r"\d+$" + + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1' + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + pt = soup.find('div', {'id' : 'pagetitle'}) + a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) + self.story.setMetadata('authorId',a['href'].split('=')[1]) + self.story.setMetadata('authorUrl','http://'+self.host+'/wrfa/'+a['href']) + self.story.setMetadata('author',a.string) + + rating=pt.text.split('(')[1].split(')')[0] + self.story.setMetadata('rating', rating) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/wrfa/'+chapter['href'])) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # <span class="label">Rated:</span> NC-17<br /> etc + content=soup.find('div',{'class' : 'content'}) + labels = soup.findAll('span',{'class':'label'}) + + value = labels[0].previousSibling + svalue = "" + while value != None: + val = value + value = value.previousSibling + while "Categories" not in str(val): + svalue += str(val) + val = val.nextSibling + self.setDescription(url,svalue) + + + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Rated' in label: + self.story.setMetadata('rating', value) + + if 'Word count' in label: + self.story.setMetadata('numWords', value.split(' -')[0]) + + if 'Categories' in label: + cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories')) + for cat in cats: + self.story.addToList('category',cat.string) + + if 'Characters' in label: + chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters')) + for char in chars: + self.story.addToList('characters',char.string) + + if 'Genre' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + if 'Complete' in label: + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + if 'Published' in label: + self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat)) + + if 'Updated' in label: + self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat)) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/wrfa/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + # can't use ^viewstory...$ in case of higher rated stories with javascript href. + storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+')) + i=1 + for a in storyas: + # skip 'report this' and 'TOC' links + if 'contact.php' not in a['href'] and 'index' not in a['href']: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulStoneSoup(self._fetchUrl(url), + selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) diff --git a/fanficdownloader/adapters/adapter_wraithbaitcom.py b/fanficdownloader/adapters/adapter_wraithbaitcom.py new file mode 100644 index 0000000..589ec22 --- /dev/null +++ b/fanficdownloader/adapters/adapter_wraithbaitcom.py @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + + +def getClass(): + return WraithBaitComAdapter + +# Class name has to be unique. Our convention is camel case the +# sitename with Adapter at the end. www is skipped. +class WraithBaitComAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + + self.decode = ["Windows-1252", + "utf8"] # 1252 is a superset of iso-8859-1. + # Most sites that claim to be + # iso-8859-1 (and some that claim to be + # utf8) are really windows-1252. + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + # get storyId from url--url validation guarantees query is only sid=1234 + self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + + + + self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId')) + + # Each adapter needs to have a unique site abbreviation. + self.story.setMetadata('siteabbrev','wb') + + # The date format will vary from site to site. + # http://docs.python.org/library/datetime.html#strftime-strptime-behavior + self.dateformat = "%d %b %Y" + + @staticmethod # must be @staticmethod, don't remove it. + def getSiteDomain(): + # The site domain. Does have www here, if it uses it. + return 'www.wraithbait.com' + + @classmethod + def getSiteExampleURLs(cls): + return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234" + + def getSiteURLPattern(self): + return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + + ## Getting the chapter list and the meta data, plus 'is adult' checking. + def extractChapterUrlsAndMetadata(self): + + if self.is_adult or self.getConfig("is_adult"): + # Weirdly, different sites use different warning numbers. + # If the title search below fails, there's a good chance + # you need a different number. print data at that point + # and see what the 'click here to continue' url says. + addurl = "&ageconsent=ok&warning=12" + else: + addurl="" + + # index=1 makes sure we see the story chapter index. Some + # sites skip that for one-chapter stories. + url = self.url+'&index=1'+addurl + logger.debug("URL: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + if "for adults only" in data: + raise exceptions.AdultCheckRequired(self.url) + + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + + # use BeautifulSoup HTML parser to make everything easier to find. + soup = bs.BeautifulSoup(data) + # print data + + # Now go hunting for all the meta data and the chapter list. + + ## Title + pt = soup.find('div', {'id' : 'pagetitle'}) + a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) + self.story.setMetadata('title',stripHTML(a)) + + # Find authorid and URL from... author url. + alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+")) + for a in alist: + self.story.addToList('authorId',a['href'].split('=')[1]) + self.story.addToList('authorUrl','http://'+self.host+'/'+a['href']) + self.story.addToList('author',a.string) + + rating=pt.text.split('[')[1].split(']')[0] + self.story.setMetadata('rating', rating) + + st = soup.find('div', {'class' : 'storytitle'}) + a = st.findAll('a', href=re.compile(r'reviews.php\?type=ST&item='+self.story.getMetadata('storyId')+"$"))[1] # second one. + self.story.setMetadata('reviews',stripHTML(a)) + + # Find the chapters: + for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")): + # include author on chapters if multiple authors. + if len(alist) > 1: + add = " by %s"%stripHTML(chapter.findNext('a', href=re.compile(r"viewuser.php\?uid=\d+"))) + else: + add = "" + # just in case there's tags, like <i> in chapter titles. + self.chapterUrls.append((stripHTML(chapter)+add,'http://'+self.host+'/'+chapter['href']+addurl)) + + self.story.setMetadata('numChapters',len(self.chapterUrls)) + + # eFiction sites don't help us out a lot with their meta data + # formating, so it's a little ugly. + + # utility method + def defaultGetattr(d,k): + try: + return d[k] + except: + return "" + + info = soup.find('div', {'class' : 'small'}) + + word=info.find(text=re.compile("Word count:")).split(':') + self.story.setMetadata('numWords', word[1]) + + cats = info.findAll('a',href=re.compile(r'browse.php\?type=categories&id=\d')) + for cat in cats: + if "General" != cat.string: + self.story.addToList('category',cat.string) + + chars = info.findAll('a',href=re.compile(r'browse.php\?type=characters&charid=\d')) + for char in chars: + self.story.addToList('characters',char.string) + + completed=info.find(text=re.compile("Completed: Yes")) + if completed != None: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + + date=soup.find('div',{'class' : 'bottom'}) + pd=date.find(text=re.compile("Published:")).string.split(': ') + self.story.setMetadata('datePublished', makeDate(stripHTML(pd[1].split(' U')[0]), self.dateformat)) + self.story.setMetadata('dateUpdated', makeDate(stripHTML(pd[2]), self.dateformat)) + + # <span class="label">Rated:</span> NC-17<br /> etc + labels = soup.findAll('span',{'class':'label'}) + pub=0 + for labelspan in labels: + value = labelspan.nextSibling + label = labelspan.string + + if 'Genres' in label: + genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) + for genre in genres: + self.story.addToList('genre',genre.string) + + if 'Warnings' in label: + warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) + for warning in warnings: + self.story.addToList('warnings',warning.string) + + try: + # Find Series name from series URL. + a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+")) + series_name = a.string + series_url = 'http://'+self.host+'/'+a['href'] + + # use BeautifulSoup HTML parser to make everything easier to find. + seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url)) + storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$')) + i=1 + for a in storyas: + if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): + self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) + break + i+=1 + + except: + # I find it hard to care if the series parsing fails + pass + + info.extract() + summary = soup.find('div', {'class' : 'content'}) + self.setDescription(url,summary) + + # grab the text for an individual chapter. + def getChapterText(self, url): + + logger.debug('Getting chapter text from: %s' % url) + + soup = bs.BeautifulSoup(self._fetchUrl(url)) + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + + return self.utf8FromSoup(url,div) + diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py new file mode 100644 index 0000000..5bf9c84 --- /dev/null +++ b/fanficdownloader/adapters/base_adapter.py @@ -0,0 +1,605 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import datetime +import time +import logging +import urllib +import urllib2 as u2 +import urlparse as up +import cookielib as cl +from functools import partial +import pickle + +import bs4 + +from ..htmlcleanup import stripHTML +from ..htmlheuristics import replace_br_with_p + +logger = logging.getLogger(__name__) + +try: + from google.appengine.api import apiproxy_stub_map + def urlfetch_timeout_hook(service, call, request, response): + if call != 'Fetch': + return + # Make the default deadline 10 seconds instead of 5. + if not request.has_deadline(): + request.set_deadline(10.0) + + apiproxy_stub_map.apiproxy.GetPreCallHooks().Append( + 'urlfetch_timeout_hook', urlfetch_timeout_hook, 'urlfetch') + logger.info("Hook to make default deadline 10.0 installed.") +except: + pass + #logger.info("Hook to make default deadline 10.0 NOT installed--not using appengine") + +from ..story import Story +from ..gziphttp import GZipProcessor +from ..configurable import Configurable +from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML +from ..exceptions import InvalidStoryURL + +try: + import chardet +except ImportError: + chardet = None + +class BaseSiteAdapter(Configurable): + + @classmethod + def matchesSite(cls,site): + return site in cls.getAcceptDomains() + + @classmethod + def getAcceptDomains(cls): + return [cls.getSiteDomain()] + + def validateURL(self): + return re.match(self.getSiteURLPattern(), self.url) + + @staticmethod + def get_empty_cookiejar(): + return cl.LWPCookieJar() + + @staticmethod + def get_empty_pagecache(): + return {} + + def __init__(self, configuration, url): + Configurable.__init__(self, configuration) + + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + self.password = "" + self.is_adult=False + + self.override_sleep = None + self.cookiejar = self.get_empty_cookiejar() + self.opener = u2.build_opener(u2.HTTPCookieProcessor(self.cookiejar),GZipProcessor()) + # self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor()) + ## Specific UA because too many sites are blocking the default python UA. + self.opener.addheaders = [('User-Agent', self.getConfig('user_agent'))] + self.storyDone = False + self.metadataDone = False + self.story = Story(configuration) + self.story.setMetadata('site',self.getConfigSection()) + self.story.setMetadata('dateCreated',datetime.datetime.now()) + self.chapterUrls = [] # tuples of (chapter title,chapter url) + self.chapterFirst = None + self.chapterLast = None + self.oldchapters = None + self.oldimgs = None + self.oldcover = None # (data of existing cover html, data of existing cover image) + self.calibrebookmark = None + self.logfile = None + + self.pagecache = self.get_empty_pagecache() + + ## order of preference for decoding. + self.decode = ["utf8", + "Windows-1252"] # 1252 is a superset of + # iso-8859-1. Most sites that + # claim to be iso-8859-1 (and + # some that claim to be utf8) + # are really windows-1252. + self._setURL(url) + if not self.validateURL(): + raise InvalidStoryURL(url, + self.getSiteDomain(), + self.getSiteExampleURLs()) + + def get_cookiejar(self): + return self.cookiejar + + def set_cookiejar(self,cj): + self.cookiejar = cj + self.opener = u2.build_opener(u2.HTTPCookieProcessor(self.cookiejar),GZipProcessor()) + self.opener.addheaders = [('User-Agent', self.getConfig('user_agent'))] + + def load_cookiejar(self,filename): + ''' + Needs to be called after adapter create, but before any fetchs + are done. Takes file *name*. + ''' + self.get_cookiejar().load(filename, ignore_discard=True, ignore_expires=True) + + # def save_cookiejar(self,filename): + # ''' + # Assumed to be a FileCookieJar if self.cookiejar set. + # Takes file *name*. + # ''' + # self.get_cookiejar().save(filename, ignore_discard=True, ignore_expires=True) + + # def save_pagecache(self,filename): + # ''' + # Writes pickle of pagecache to file *name* + # ''' + # with open(filename, 'wb') as f: + # pickle.dump(self.get_pagecache(), + # f,protocol=pickle.HIGHEST_PROTOCOL) + + # def load_pagecache(self,filename): + # ''' + # Reads pickle of pagecache from file *name* + # ''' + # with open(filename, 'rb') as f: + # self.set_pagecache(pickle.load(f)) + + def get_pagecache(self): + return self.pagecache + + def set_pagecache(self,d): + self.pagecache=d + + def _get_cachekey(self, url, parameters=None, headers=None): + keylist=[url] + if parameters != None: + keylist.append('&'.join('{0}={1}'.format(key, val) for key, val in sorted(parameters.items()))) + if headers != None: + keylist.append('&'.join('{0}={1}'.format(key, val) for key, val in sorted(headers.items()))) + return '?'.join(keylist) + + def _has_cachekey(self,cachekey): + return self.use_pagecache() and cachekey in self.get_pagecache() + + def _get_from_pagecache(self,cachekey): + if self.use_pagecache(): + return self.get_pagecache().get(cachekey) + else: + return None + + def _set_to_pagecache(self,cachekey,data): + if self.use_pagecache(): + self.get_pagecache()[cachekey] = data + + def use_pagecache(self): + ''' + adapters that will work with the page cache need to implement + this and change it to True. + ''' + return False + + # def story_load(self,filename): + # d = pickle.load(self.story.metadata,filename) + # self.story.metadata = d['metadata'] + # self.chapterUrls = d['chapterlist'] + # self.story.metadataDone = True + + def _setURL(self,url): + self.url = url + self.parsedUrl = up.urlparse(url) + self.host = self.parsedUrl.netloc + self.path = self.parsedUrl.path + self.story.setMetadata('storyUrl',self.url) + +## website encoding(s)--in theory, each website reports the character +## encoding they use for each page. In practice, some sites report it +## incorrectly. Each adapter has a default list, usually "utf8, +## Windows-1252" or "Windows-1252, utf8". The special value 'auto' +## will call chardet and use the encoding it reports if it has +90% +## confidence. 'auto' is not reliable. + def _decode(self,data): + if self.getConfig('website_encodings'): + decode = self.getConfigList('website_encodings') + else: + decode = self.decode + + for code in decode: + try: + #print code + if code == "auto": + if not chardet: + logger.info("chardet not available, skipping 'auto' encoding") + continue + detected = chardet.detect(data) + #print detected + if detected['confidence'] > 0.9: + code=detected['encoding'] + else: + continue + return data.decode(code) + except: + logger.debug("code failed:"+code) + pass + logger.info("Could not decode story, tried:%s Stripping non-ASCII."%decode) + return "".join([x for x in data if ord(x) < 128]) + + # Assumes application/x-www-form-urlencoded. parameters, headers are dict()s + def _postUrl(self, url, + parameters={}, + headers={}, + extrasleep=None, + usecache=True): + ''' + When should cache be cleared or not used? logins... + + extrasleep is primarily for ffnet adapter which has extra + sleeps. Passed into fetchs so it can be bypassed when + cache hits. + ''' + cachekey=self._get_cachekey(url, parameters, headers) + if usecache and self._has_cachekey(cachekey): + logger.debug("#####################################\npagecache HIT: %s"%cachekey) + return self._get_from_pagecache(cachekey) + + logger.debug("#####################################\npagecache MISS: %s"%cachekey) + self.do_sleep(extrasleep) + + ## u2.Request assumes POST when data!=None. Also assumes data + ## is application/x-www-form-urlencoded. + if 'Content-type' not in headers: + headers['Content-type']='application/x-www-form-urlencoded' + if 'Accept' not in headers: + headers['Accept']="text/html,*/*" + req = u2.Request(url, + data=urllib.urlencode(parameters), + headers=headers) + data = self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()) + self._set_to_pagecache(cachekey,data) + return data + + def _fetchUrlRaw(self, url, + parameters=None, + extrasleep=None, + usecache=True): + ''' + When should cache be cleared or not used? logins... + + extrasleep is primarily for ffnet adapter which has extra + sleeps. Passed into fetchs so it can be bypassed when + cache hits. + ''' + cachekey=self._get_cachekey(url, parameters) + if usecache and self._has_cachekey(cachekey): + logger.debug("#####################################\npagecache HIT: %s"%cachekey) + return self._get_from_pagecache(cachekey) + + logger.debug("#####################################\npagecache MISS: %s"%cachekey) + self.do_sleep(extrasleep) + if parameters != None: + data = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read() + else: + data = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read() + self._set_to_pagecache(cachekey,data) + return data + + def set_sleep(self,val): + print("\n===========\n set sleep time %s\n==========="%val) + self.override_sleep = val + + def do_sleep(self,extrasleep=None): + if extrasleep: + time.sleep(float(extrasleep)) + if self.override_sleep: + time.sleep(float(self.override_sleep)) + elif self.getConfig('slow_down_sleep_time'): + time.sleep(float(self.getConfig('slow_down_sleep_time'))) + + # parameters is a dict() + def _fetchUrl(self, url, + parameters=None, + usecache=True, + extrasleep=None): + + excpt=None + for sleeptime in [0, 0.5, 4, 9]: + time.sleep(sleeptime) + try: + return self._decode(self._fetchUrlRaw(url, + parameters=parameters, + usecache=usecache, + extrasleep=extrasleep)) + except u2.HTTPError, he: + excpt=he + if he.code == 404: + logger.warn("Caught an exception reading URL: %s Exception %s."%(unicode(url),unicode(he))) + break # break out on 404 + # except Exception, e: + # excpt=e + # logger.warn("Caught an exception reading URL: %s Exception %s."%(unicode(url),unicode(e))) + + logger.error("Giving up on %s" %url) + logger.exception(excpt) + raise(excpt) + + # Limit chapters to download. Input starts at 1, list starts at 0 + def setChaptersRange(self,first=None,last=None): + if first: + self.chapterFirst=int(first)-1 + if last: + self.chapterLast=int(last)-1 + + # Does the download the first time it's called. + def getStory(self): + if not self.storyDone: + self.getStoryMetadataOnly(get_cover=True) + + for index, (title,url) in enumerate(self.chapterUrls): + if (self.chapterFirst!=None and index < self.chapterFirst) or \ + (self.chapterLast!=None and index > self.chapterLast): + self.story.addChapter(url, + removeEntities(title), + None) + else: + if self.oldchapters and index < len(self.oldchapters): + data = self.utf8FromSoup(None, + self.oldchapters[index], + partial(cachedfetch,self._fetchUrlRaw,self.oldimgs)) + else: + data = self.getChapterText(url) + self.story.addChapter(url, + removeEntities(title), + removeEntities(data)) + self.storyDone = True + + # include image, but no cover from story, add default_cover_image cover. + if self.getConfig('include_images') and \ + not self.story.cover and \ + self.getConfig('default_cover_image'): + self.story.addImgUrl(None, + #self.getConfig('default_cover_image'), + self.story.formatFileName(self.getConfig('default_cover_image'), + self.getConfig('allow_unsafe_filename')), + self._fetchUrlRaw, + cover=True) + + # no new cover, set old cover, if there is one. + if not self.story.cover and self.oldcover: + self.story.oldcover = self.oldcover + + # cheesy way to carry calibre bookmark file forward across update. + if self.calibrebookmark: + self.story.calibrebookmark = self.calibrebookmark + if self.logfile: + self.story.logfile = self.logfile + + return self.story + + def getStoryMetadataOnly(self,get_cover=True): + if not self.metadataDone: + self.doExtractChapterUrlsAndMetadata(get_cover=get_cover) + + if not self.story.getMetadataRaw('dateUpdated'): + self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished')) + + self.metadataDone = True + return self.story + + def hookForUpdates(self,chaptercount): + "Usually not needed." + return chaptercount + + ############################### + + @staticmethod + def getSiteDomain(): + "Needs to be overriden in each adapter class." + return 'no such domain' + + @classmethod + def getConfigSection(cls): + "Only needs to be overriden if != site domain." + return cls.getSiteDomain() + + @classmethod + def stripURLParameters(cls,url): + "Only needs to be overriden if URL contains more than one parameter" + ## remove any trailing '&' parameters--?sid=999 will be left. + ## that's all that any of the current adapters need or want. + return re.sub(r"&.*$","",url) + + ## URL pattern validation is done *after* picking an adaptor based + ## on domain instead of *as* the adaptor selector so we can offer + ## the user example(s) for that particular site. + ## Override validateURL(self) instead if you need more control. + def getSiteURLPattern(self): + "Used to validate URL. Should be override in each adapter class." + return '^http://'+re.escape(self.getSiteDomain()) + + @classmethod + def getSiteExampleURLs(cls): + """ + Return a string of space separated example URLs. + Needs to be overriden in each adapter class. It's the adapter + writer's responsibility to make sure the example(s) pass the + validateURL method. + """ + return 'no such example' + + def doExtractChapterUrlsAndMetadata(self,get_cover=True): + ''' + There are a handful of adapters that fetch a cover image while + collecting metadata. That isn't needed while *just* + collecting metadata in FG in plugin. Those few will override + this instead of extractChapterUrlsAndMetadata() + ''' + return self.extractChapterUrlsAndMetadata() + + def extractChapterUrlsAndMetadata(self): + "Needs to be overriden in each adapter class. Populates self.story metadata and self.chapterUrls" + pass + + def getChapterText(self, url): + "Needs to be overriden in each adapter class." + pass + + # Just for series, in case we choose to change how it's stored or represented later. + def setSeries(self,name,num): + if self.getConfig('collect_series'): + self.story.setMetadata('series','%s [%s]'%(name, int(num))) + + def setDescription(self,url,svalue): + #print("\n\nsvalue:\n%s\n"%svalue) + if self.getConfig('keep_summary_html'): + if isinstance(svalue,basestring): + # bs4/html5lib add html, header and body tags, which + # we don't want. utf8FromSoup will strip the body tags for us. + svalue = bs4.BeautifulSoup(svalue,"html5lib").body + self.story.setMetadata('description',self.utf8FromSoup(url,svalue)) + else: + self.story.setMetadata('description',stripHTML(svalue)) + #print("\n\ndescription:\n"+self.story.getMetadata('description')+"\n\n") + + def setCoverImage(self,storyurl,imgurl): + if self.getConfig('include_images'): + self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True, + coverexclusion=self.getConfig('cover_exclusion_regexp')) + + # bs3 & bs4 are different here. + # will move to a bs3 vs bs4 block if there's lots of changes. + def get_attr_keys(self,soup): + if hasattr(soup, '_getAttrMap') and getattr(soup, '_getAttrMap') is not None: + # bs3 + #print "bs3 attrs:%s"%soup._getAttrMap().keys() + return soup._getAttrMap().keys() + elif hasattr(soup, 'attrs') and isinstance(soup.attrs,dict): + #print "bs4 attrs:%s"%soup.attrs.keys() + # bs4 + return soup.attrs.keys() + return [] + + # This gives us a unicode object, not just a string containing bytes. + # (I gave soup a unicode string, you'd think it could give it back...) + # Now also does a bunch of other common processing for us. + def utf8FromSoup(self,url,soup,fetch=None): + if not fetch: + fetch=self._fetchUrlRaw + + acceptable_attributes = ['href','name','class','id'] + if self.getConfig("keep_style_attr"): + acceptable_attributes.append('style') + #print("include_images:"+self.getConfig('include_images')) + if self.getConfig('include_images'): + acceptable_attributes.extend(('src','alt','longdesc')) + for img in soup.findAll('img'): + # some pre-existing epubs have img tags that had src stripped off. + if img.has_key('src'): + (img['src'],img['longdesc'])=self.story.addImgUrl(url,img['src'],fetch, + coverexclusion=self.getConfig('cover_exclusion_regexp')) + + for attr in self.get_attr_keys(soup): + if attr not in acceptable_attributes: + del soup[attr] ## strip all tag attributes except href and name + + try: + # as a generator, each tag will be returned even if there's a + # mismatch at the end. + for t in soup.recursiveChildGenerator(): # findAll(recursive=True): + for attr in self.get_attr_keys(t): + if attr not in acceptable_attributes: + del t[attr] ## strip all tag attributes except acceptable_attributes + + # these are not acceptable strict XHTML. But we do already have + # CSS classes of the same names defined + if t and hasattr(t,'name') and t.name is not None: + if t.name in ('u'): + t['class']=t.name + t.name='span' + if t.name in ('center'): + t['class']=t.name + t.name='div' + # removes paired, but empty non paragraph tags. + if t.name not in ('p') and t.string != None and len(t.string.strip()) == 0 : + t.extract() + except AttributeError, ae: + print("Error parsing HTML, probably poor input HTML. %s"%ae) + + retval = unicode(soup) + + if self.getConfig('nook_img_fix') and not self.getConfig('replace_br_with_p'): + # if the <img> tag doesn't have a div or a p around it, + # nook gets confused and displays it on every page after + # that under the text for the rest of the chapter. + retval = re.sub(r"(?!<(div|p)>)\s*(?P<imgtag><img[^>]+>)\s*(?!</(div|p)>)", + "<div>\g<imgtag></div>",retval) + + # Don't want body tags in chapter html--writers add them. + # This is primarily for epub updates. + retval = re.sub(r"</?body[^>]*>\r?\n?","",retval) + + if self.getConfig("replace_br_with_p"): + # Apply heuristic processing to replace <br> paragraph + # breaks with <p> tags. + retval = replace_br_with_p(retval) + + if self.getConfig('replace_hr'): + # replacing a self-closing tag with a container tag in the + # soup is more difficult than it first appears. So cheat. + retval = retval.replace("<hr />","<div class='center'>* * *</div>") + + return retval + + def make_soup(self,data): + ''' + Convenience method for getting a bs4 soup. Older and + non-updated adapters call the included bs3 library themselves. + ''' + return bs4.BeautifulSoup(data,'html5lib') + +def cachedfetch(realfetch,cache,url): + if url in cache: + return cache[url] + else: + return realfetch(url) + +fullmon = {u"January":u"01", u"February":u"02", u"March":u"03", u"April":u"04", u"May":u"05", + u"June":u"06","July":u"07", u"August":u"08", u"September":u"09", u"October":u"10", + u"November":u"11", u"December":u"12" } + +def makeDate(string,dateform): + # Surprise! Abstracting this turned out to be more useful than + # just saving bytes. + + # fudge english month names for people who's locale is set to + # non-english. All our current sites date in english, even if + # there's non-english content. -- ficbook.net now makes that a + # lie. It has to do something even more complicated to get + # Russian month names correct everywhere. + do_abbrev = "%b" in dateform + + if u"%B" in dateform or do_abbrev: + dateform = dateform.replace(u"%B",u"%m").replace(u"%b",u"%m") + for (name,num) in fullmon.items(): + if do_abbrev: + name = name[:3] # first three for abbrev + if name in string: + string = string.replace(name,num) + break + + return datetime.datetime.strptime(string.encode('utf-8'),dateform.encode('utf-8')) + diff --git a/fanficdownloader/adapters/base_efiction_adapter.py b/fanficdownloader/adapters/base_efiction_adapter.py new file mode 100644 index 0000000..403a28a --- /dev/null +++ b/fanficdownloader/adapters/base_efiction_adapter.py @@ -0,0 +1,426 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Software: eFiction +# import time +# import urllib +import logging +logger = logging.getLogger(__name__) +import re +import urllib2 + +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions + +from base_adapter import BaseSiteAdapter, makeDate + +""" +This is a generic adapter for eFiction based archives (see +http://fanlore.org/wiki/List_of_eFiction_Archives for a list). + +Most of them share common traits: + * No HTTPS + * 'www.' is optional + * Default story template is 'viewstory.php' with arguments + * 'sid' the storyId + * 'chapter' for chapters (will be thrown away anyway by + stripURLParameters in base_adapter + Use Printable version which is easier to parse and has everything in one + page and cache between extractChapterUrlsAndMetadata and getChapterText +""" + +# PHP constants +_RUSERSONLY = 'Registered Users Only' +_NOSUCHACCOUNT = "There is no such account on our website" +_WRONGPASSWORD = "That password doesn't match the one in our database" +_USERACCOUNT = 'Member Account' + +# Regular expressions +_REGEX_WARING_PARAM = re.compile("warning=(?P<warningId>\d+)") +_REGEX_CHAPTER_B = re.compile("^(?P<chapterId>\d+)\.") +_REGEX_CHAPTER_PARAM = re.compile("chapter=(?P<chapterId>\d+)$") +_REGEX_CHAPTER_FRAGMENT = re.compile("^#(?P<chapterId>\d+)$") +_REGEX_DOESNT_START_WITH_HTTP = re.compile("^(?!http)") + +class BaseEfictionAdapter(BaseSiteAdapter): + + def __init__(self, config, url): + BaseSiteAdapter.__init__(self, config, url) + self.story.setMetadata('siteabbrev',self.getSiteAbbrev()) + self.decode = self.getEncoding() + storyId = re.compile(self.getSiteURLPattern()).match(self.url).group('storyId') + self.story.setMetadata('storyId', storyId) + self._setURL(self.getViewStoryUrl(storyId)) + self.triedLoggingIn = False + self.triedAcceptWarnings = False + self.username = "NoneGiven" # if left empty, site doesn't return any message at all. + + @classmethod + def getAcceptDomains(cls): + return [cls.getSiteDomain(),'www.' + cls.getSiteDomain()] + + @classmethod + def getSiteExampleURLs(cls): + return cls.getViewStoryUrl('1234') + ' ' + cls.getViewStoryUrl('1234') + '&chapter=2' + + @classmethod + def getSiteURLPattern(self): + return r"http://(www\.)?%s%s/%s\?sid=(?P<storyId>\d+)" % (self.getSiteDomain(), self.getPathToArchive(), self.getViewStoryPhpName()) + + @classmethod + def getEncoding(cls): + """ + Return an array of character encodings to try to decode the HTML with + """ + return ["Windows-1252", "utf8"] + + + @classmethod + def getPathToArchive(cls): + """ + Get the path segment of the archive, default '/'. + + In many cases, it's '/archive' or '/fanfiction' + """ + return "" + + @classmethod + def getViewStoryPhpName(cls): + """ + Get the name of the story PHP script, by default 'viewstory.php' + """ + return "viewstory.php" + + @classmethod + def getViewUserPhpName(cls): + """ + Get the name of the user PHP script, by default 'viewuser.php' + """ + return "viewuser.php" + + @classmethod + def getUserPhpName(cls): + """ + Get the name of the user PHP script, by default 'viewuser.php' + """ + return "user.php" + + @classmethod + def getDateFormat(self): + """ + Describe the date format of this site in terms of strftime + See http://docs.python.org/library/datetime.html#strftime-strptime-behavior + """ + return "%d %b %Y" + + @classmethod + def getUrlForPhp(self, php): + return "http://%s%s/%s" % (self.getSiteDomain(), self.getPathToArchive(), php) + + @classmethod + def getViewStoryUrl(self, storyId): + """ + Get the URL to a user page on this site. + """ + return "%s?sid=%s" % (self.getUrlForPhp(self.getViewStoryPhpName()), storyId) + + @classmethod + def getViewUserUrl(self, userId): + """ + Get the URL to a user page on this site. + """ + return "%s?sid=%s" % (self.getUrlForPhp(self.getViewUserPhpName()), userId) + + @classmethod + def getLoginUrl(self): + """ + Get the URL to the login page on this site. + """ + return "%s?action=login" % self.getUrlForPhp(self.getUserPhpName()) + + @classmethod + def getMessageRegisteredUsersOnly(self): + """ + Constant _RUSERSONLY defined in languages/en.php + """ + return _RUSERSONLY + + @classmethod + def getMessageThereIsNoSuchAccount(self): + """ + Constant _NOSUCHACCOUNT defined in languages/en.php + """ + return _NOSUCHACCOUNT + + @classmethod + def getMessageWrongPassword(self): + """ + Constant _WRONGPASSWORD defined in languages/en.php + """ + return _WRONGPASSWORD + + @classmethod + def getMessageMemberAccount(self): + """ + Constant _USERACCOUNT defined in languages/en.php + """ + return _USERACCOUNT + + ## Login seems to be reasonably standard across eFiction sites. + @classmethod + def needToLoginCheck(self, html): + """ + Return whether the HTML contains either of _RUSERSONLY, _NOSUCHACCOUNT or _WRONGPASSWORD + """ + return getMessageRegisteredUsersOnly() in html \ + or getMessageThereIsNoSuchAccount in html \ + or getMessageWrongPassword in html + + def _fetch_to_soup(self, url): + """ + Fetch a HTML document, fix it and parse it to BeautifulSoup. + + Replaces old characters, broken meta-tags, non-self-closing hr/br. + + Makes image links absolute so they can be downloaded + """ + try: + html = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + + # Some site use old, old-school Comments <!- comment -> (single dash) + html = re.sub("<!-.+?->", "", html) + + # There is a problem with meta tags on some sites where spaces aren't + # properly encoded + html = re.sub("<meta[^<>]+>(.*</meta>)?", "", html) + + # fix non-closing hr/br + html = html.replace("<hr>", "<hr/>") + html = html.replace("<br>", "<br/>") + + soup = bs.BeautifulSoup(html, selfClosingTags=['br','hr']) # otherwise soup eats the br/hr tags.) + + ## fix all local image 'src' to absolute + for img in soup.findAll("img", {"src": _REGEX_DOESNT_START_WITH_HTTP}): + # TODO handle '../../' and so on + if img['src'].startswith('/'): + img['src'] = img['src'][1:] + img['src'] = "http://%s%s/%s" % (self.getSiteDomain(), self.getPathToArchive(), img['src']) + + return soup + + def performLogin(self, url): + params = {} + + if self.password: + params['penname'] = self.username + params['password'] = self.password + else: + params['penname'] = self.getConfig("username") + params['password'] = self.getConfig("password") + params['cookiecheck'] = '1' + params['submit'] = 'Submit' + + logger.debug("Will now login to URL (%s) as (%s)" % (self.getLoginUrl(), params['penname'])) + + d = self._fetchUrl(self.getLoginUrl(), params) + + if self.getMessageMemberAccount() not in d : #Member Account + logger.info("Failed to login to URL <%s> as '%s'" % (self.getLoginUrl(), params['penname'])) + raise exceptions.FailedToLogin(url, params['penname']) + return False + else: + return True + + def handleMetadataPair(self, key, value): + """ + Handles a key-value pair of story metadata. + + Returns straight away if the value is 'None' (that's a string) + + Can be overridden by subclasses:: + def handleMetadataPair(self, key, value): + if key == 'MyCustomKey': + self.story.setMetadata('somekye', value) + else: + super(NameOfMyAdapter, self).handleMetadata(key, value) + """ + # logger.debug("metadata: '%s' == '%s'" % (key, value)) + if value == 'None': + return + elif key == 'Summary': + self.setDescription(self.url, value) + elif 'Genre' in key: + for val in re.split("\s*,\s*", value): + self.story.addToList('genre', val) + elif 'Warning' in key: + for val in re.split("\s*,\s*", value): + self.story.addToList('warnings', val) + elif 'Characters' in key: + for val in re.split("\s*,\s*", value): + self.story.addToList('characters', val) + elif 'Categories' in key: + for val in re.split("\s*,\s*", value): + self.story.addToList('category', val) + elif 'Challenges' in key: + for val in re.split("\s*,\s*", value): + # TODO this should be an official field I guess + self.story.addToList('challenge', val) + elif key == 'Chapters': + self.story.setMetadata('numChapters', int(value)) + elif key == 'Rating': + self.story.setMetadata('rating', value) + elif key == 'Word count': + self.story.setMetadata('numWords', value) + elif key == 'Completed': + if 'Yes' in value: + self.story.setMetadata('status', 'Completed') + else: + self.story.setMetadata('status', 'In-Progress') + elif key == 'Read': + # TODO this should be an official field I guess + self.story.setMetadata('readings', value) + elif key == 'Published': + self.story.setMetadata('datePublished', makeDate(value, self.getDateFormat())) + elif key == 'Updated': + self.story.setMetadata('dateUpdated', makeDate(value, self.getDateFormat())) + elif key == 'Pairing': + for val in re.split("\s*,\s*", value): + self.story.addToList('ships', val) + elif key == 'Series': + ## TODO is not a link in the printable view, so no seriesURL possible + self.story.setMetadata('series', value) + else: + logger.info("Unhandled metadata pair: '%s' : '%s'" % (key, value)) + + def extractChapterUrlsAndMetadata(self): + printUrl = self.url + '&action=printable&textsize=0&chapter=' + if self.getConfig('bulk_load'): + printUrl += 'all' + else: + printUrl += '1' + + + soup = self._fetch_to_soup(printUrl) + + ## Handle warnings and login checks + errorDiv = soup.find("div", "errortext") + while errorDiv is not None: + if self.getMessageRegisteredUsersOnly() in errorDiv.prettify(): + if not self.triedLoggingIn: + self.performLogin(self.url) + soup = self._fetch_to_soup(printUrl) + errorDiv = soup.find("div", "errortext") + self.triedLoggingIn = True + else: + raise exceptions.FailedToLogin(self.url, str(errorDiv)) + else: + warningLink = errorDiv.find("a") + if warningLink is not None and ( \ + 'ageconsent' in warningLink['href'] \ + or 'warning' in warningLink['href']): + if not self.triedAcceptWarnings: + if not (self.is_adult or self.getConfig("is_adult")): + raise exceptions.AdultCheckRequired(self.url) + # XXX Using this method, we're independent of # getHighestWarningLevel + printUrl += "&ageconsent=ok&warning=%s" % (_REGEX_WARING_PARAM.search(warningLink['href']).group(1)) + # printUrl += "&ageconsent=ok&warning=%s" % self.getHighestWarningLevel() + soup = self._fetch_to_soup(printUrl) + errorDiv = soup.find("div", "errortext") + self.triedAcceptWarnings = True + else: + raise exception.FailedToDownload(self.url, str(errorDiv)) + else: + raise exception.FailedToDownload(self.url, str(errorDiv)) + + # title and author + pagetitleDiv = soup.find("div", {"id": "pagetitle"}) + if pagetitleDiv.find('a') is None: + raise execeptions.FailedToDownload("Couldn't find title and author") + self.story.setMetadata('title', pagetitleDiv.find("a").text) + authorLink = pagetitleDiv.findAll("a")[1] + self.story.setMetadata('author', authorLink.text) + self.story.setMetadata('authorId', re.search("\d+", authorLink['href']).group(0)) + self.story.setMetadata('authorUrl', self.getViewUserUrl(self.story.getMetadata('authorId'))) + + ## Parse the infobox + labelSpans = soup.find("div", "infobox").find("div", "content").findAll("span", "label") + for labelSpan in labelSpans: + valueStr = "" + nextEl = labelSpan.nextSibling + while nextEl is not None and not (\ + type(nextEl) is bs.Tag \ + and nextEl.name == "span" \ + and nextEl['class'] =='label' \ + ): + ## must string copy nextEl or nextEl will change trees + if (type(nextEl) is bs.Tag): + valueStr += nextEl.prettify() + else: + valueStr += str(nextEl) + nextEl = nextEl.nextSibling + key = labelSpan.text.strip() + + ## strip trailing line breaks + valueStr = re.sub("<br />", "", valueStr) + + ## strip trailing colons + key = re.sub("\s*:\s*$", "", key) + + ## strip whitespace + key = key.strip() + valueStr = stripHTML(valueStr) + + self.handleMetadataPair(key, valueStr) + + ## Chapter URLs + + # If we didn't bulk-load the whole chapter we now need to load + # the non-printable HTML version of the landing page (i.e. the story + # URL to get the Chapter titles + if not self.getConfig('bulk_load'): + soup = self._fetch_to_soup(self.url + '&index=1') + + chapterLinks = [] + for b in soup.findAll("b", text=_REGEX_CHAPTER_B): + chapterId = _REGEX_CHAPTER_B.search(b).group('chapterId') + chapterLink = b.findNext("a") + chapterLink['href'] = "%s&chapter=%s" % (self.url, chapterId) + self.chapterUrls.append((chapterLink.text, chapterLink['href'])) + + ## Store reference to soup for getChapterText + self.html = soup + + def getChapterText(self, url): + if self.getConfig('bulk_load'): + logger.debug('Cached chapter text from <%s>' % url) + anchor = _REGEX_CHAPTER_PARAM.search(url).group(1) + chapterDiv = self.html.find("a", {"name": anchor}).parent.findNext("div", "chapter") + else: + logger.debug('Download chapter text from <%s>' % url) + soup = self._fetch_to_soup(url + '&action=printable') + chapterDiv = soup.find("div", "chapter") + return self.utf8FromSoup(self.url, chapterDiv) + +def getClass(): + return BaseEfictionAdapter diff --git a/fanficdownloader/configurable.py b/fanficdownloader/configurable.py new file mode 100644 index 0000000..9b6d35f --- /dev/null +++ b/fanficdownloader/configurable.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ConfigParser, re + +# All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc) +# inherit from Configurable. The config file(s) uses ini format: +# [sections] with key:value settings. +# +# [defaults] +# titlepage_entries: category,genre, status +# [www.whofic.com] +# titlepage_entries: category,genre, status,dateUpdated,rating +# [epub] +# titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated +# [www.whofic.com:epub] +# titlepage_entries: category,genre, status,datePublished +# [overrides] +# titlepage_entries: category + +class Configuration(ConfigParser.SafeConfigParser): + + def __init__(self, site, fileform): + ConfigParser.SafeConfigParser.__init__(self) + self.sectionslist = ['defaults'] + + if site.startswith("www."): + sitewith = site + sitewithout = site.replace("www.","") + else: + sitewith = "www."+site + sitewithout = site + + self.addConfigSection(sitewith) + self.addConfigSection(sitewithout) + if fileform: + self.addConfigSection(fileform) + self.addConfigSection(sitewith+":"+fileform) + self.addConfigSection(sitewithout+":"+fileform) + self.addConfigSection("overrides") + + self.listTypeEntries = [ + 'category', + 'genre', + 'characters', + 'ships', + 'warnings', + 'extratags', + 'author', + 'authorId', + 'authorUrl', + 'lastupdate', + ] + + self.validEntries = self.listTypeEntries + [ + 'series', + 'seriesUrl', + 'language', + 'status', + 'datePublished', + 'dateUpdated', + 'dateCreated', + 'rating', + 'numChapters', + 'numWords', + 'site', + 'storyId', + 'title', + 'storyUrl', + 'description', + 'formatname', + 'formatext', + 'siteabbrev', + 'version', + # internal stuff. + 'authorHTML', + 'seriesHTML', + 'langcode', + 'output_css', + ] + + def addConfigSection(self,section): + self.sectionslist.insert(0,section) + + def isListType(self,key): + return key in self.listTypeEntries or self.hasConfig("include_in_"+key) + + def isValidMetaEntry(self, key): + return key in self.getValidMetaList() + + def getValidMetaList(self): + return self.validEntries + self.getConfigList("extra_valid_entries") + + # used by adapters & writers, non-convention naming style + def hasConfig(self, key): + return self.has_config(self.sectionslist, key) + + def has_config(self, sections, key): + for section in sections: + try: + self.get(section,key) + #print("found %s in section [%s]"%(key,section)) + return True + except: + try: + self.get(section,"add_to_"+key) + #print("found add_to_%s in section [%s]"%(key,section)) + return True + except: + pass + + return False + + # used by adapters & writers, non-convention naming style + def getConfig(self, key, default=""): + return self.get_config(self.sectionslist,key,default) + + def get_config(self, sections, key, default=""): + val = default + for section in sections: + try: + val = self.get(section,key) + if val and val.lower() == "false": + val = False + #print "getConfig(%s)=[%s]%s" % (key,section,val) + break + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e: + pass + + for section in sections[::-1]: + # 'martian smiley' [::-1] reverses list by slicing whole list with -1 step. + try: + val = val + self.get(section,"add_to_"+key) + #print "getConfig(add_to_%s)=[%s]%s" % (key,section,val) + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e: + pass + + return val + + # split and strip each. + def get_config_list(self, sections, key): + vlist = re.split(r'(?<!\\),',self.get_config(sections,key)) # don't split on \, + vlist = filter( lambda x : x !='', [ v.strip().replace('\,',',') for v in vlist ]) + #print "vlist("+key+"):"+str(vlist) + return vlist + + # used by adapters & writers, non-convention naming style + def getConfigList(self, key): + return self.get_config_list(self.sectionslist, key) + +# extended by adapter, writer and story for ease of calling configuration. +class Configurable(object): + + def __init__(self, configuration): + self.configuration = configuration + + def isListType(self,key): + return self.configuration.isListType(key) + + def isValidMetaEntry(self, key): + return self.configuration.isValidMetaEntry(key) + + def getValidMetaList(self): + return self.configuration.getValidMetaList() + + def hasConfig(self, key): + return self.configuration.hasConfig(key) + + def has_config(self, sections, key): + return self.configuration.has_config(sections, key) + + def getConfig(self, key, default=""): + return self.configuration.getConfig(key,default) + + def get_config(self, sections, key, default=""): + return self.configuration.get_config(sections,key,default) + + def getConfigList(self, key): + return self.configuration.getConfigList(key) + + def get_config_list(self, sections, key): + return self.configuration.get_config_list(sections,key) diff --git a/fanficdownloader/epubutils.py b/fanficdownloader/epubutils.py new file mode 100644 index 0000000..2e1b041 --- /dev/null +++ b/fanficdownloader/epubutils.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2014, Jim Miller' +__docformat__ = 'restructuredtext en' + +import logging +logger = logging.getLogger(__name__) + +import re, os, traceback +from zipfile import ZipFile +from xml.dom.minidom import parseString + +import bs4 as bs + +def get_dcsource(inputio): + return get_update_data(inputio,getfilecount=False,getsoups=False)[0] + +def get_dcsource_chaptercount(inputio): + return get_update_data(inputio,getfilecount=True,getsoups=False)[:2] # (source,filecount) + +def get_update_data(inputio, + getfilecount=True, + getsoups=True): + epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob + + ## Find the .opf file. + container = epub.read("META-INF/container.xml") + containerdom = parseString(container) + rootfilenodelist = containerdom.getElementsByTagName("rootfile") + rootfilename = rootfilenodelist[0].getAttribute("full-path") + + contentdom = parseString(epub.read(rootfilename)) + firstmetadom = contentdom.getElementsByTagName("metadata")[0] + try: + source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8") + except: + source=None + + ## Save the path to the .opf file--hrefs inside it are relative to it. + relpath = get_path_part(rootfilename) + + oldcover = None + calibrebookmark = None + logfile = None + # Looking for pre-existing cover. + for item in contentdom.getElementsByTagName("reference"): + if item.getAttribute("type") == "cover": + # there is a cover (x)html file, save the soup for it. + href=relpath+item.getAttribute("href") + oldcoverhtmlhref = href + oldcoverhtmldata = epub.read(href) + oldcoverhtmltype = "application/xhtml+xml" + for item in contentdom.getElementsByTagName("item"): + if( relpath+item.getAttribute("href") == oldcoverhtmlhref ): + oldcoverhtmltype = item.getAttribute("media-type") + break + soup = bs.BeautifulSoup(oldcoverhtmldata.decode("utf-8")) + src = None + # first img or image tag. + imgs = soup.findAll('img') + if imgs: + src = get_path_part(href)+imgs[0]['src'] + else: + imgs = soup.findAll('image') + if imgs: + src=get_path_part(href)+imgs[0]['xlink:href'] + + if not src: + continue + try: + # remove all .. and the path part above it, if present. + # Mostly for epubs edited by Sigil. + src = re.sub(r"([^/]+/\.\./)","",src) + #print("epubutils: found pre-existing cover image:%s"%src) + oldcoverimghref = src + oldcoverimgdata = epub.read(src) + for item in contentdom.getElementsByTagName("item"): + if( relpath+item.getAttribute("href") == oldcoverimghref ): + oldcoverimgtype = item.getAttribute("media-type") + break + oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata) + except Exception as e: + logger.warn("Cover Image %s not found"%src) + logger.warn("Exception: %s"%(unicode(e))) + traceback.print_exc() + + filecount = 0 + soups = [] # list of xhmtl blocks + images = {} # dict() longdesc->data + if getfilecount: + # spin through the manifest--only place there are item tags. + for item in contentdom.getElementsByTagName("item"): + # First, count the 'chapter' files. FFDL uses file0000.xhtml, + # but can also update epubs downloaded from Twisting the + # Hellmouth, which uses chapter0.html. + if( item.getAttribute("media-type") == "application/xhtml+xml" ): + href=relpath+item.getAttribute("href") + #print("---- item href:%s path part: %s"%(href,get_path_part(href))) + if re.match(r'.*/log_page\.x?html',href): + try: + logfile = epub.read(href).decode("utf-8") + except: + pass # corner case I bumped into while testing. + if re.match(r'.*/(file|chapter)\d+\.x?html',href): + if getsoups: + soup = bs.BeautifulSoup(epub.read(href).decode("utf-8")) + for img in soup.findAll('img'): + newsrc='' + longdesc='' + try: + newsrc=get_path_part(href)+img['src'] + # remove all .. and the path part above it, if present. + # Mostly for epubs edited by Sigil. + newsrc = re.sub(r"([^/]+/\.\./)","",newsrc) + longdesc=img['longdesc'] + data = epub.read(newsrc) + images[longdesc] = data + img['src'] = img['longdesc'] + except Exception as e: + logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc)) + logger.warn("Exception: %s"%(unicode(e))) + traceback.print_exc() + soup = soup.find('body') + # ffdl epubs have chapter title h3 + h3 = soup.find('h3') + if h3: + h3.extract() + # TtH epubs have chapter title h2 + h2 = soup.find('h2') + if h2: + h2.extract() + + for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}): + skip.extract() + + soups.append(soup) + + filecount+=1 + + try: + calibrebookmark = epub.read("META-INF/calibre_bookmarks.txt") + except: + pass + + #for k in images.keys(): + #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k]))) + return (source,filecount,soups,images,oldcover,calibrebookmark,logfile) + +def get_path_part(n): + relpath = os.path.dirname(n) + if( len(relpath) > 0 ): + relpath=relpath+"/" + return relpath + +def get_story_url_from_html(inputio,_is_good_url=None): + + #print("get_story_url_from_html called") + epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob + + ## Find the .opf file. + container = epub.read("META-INF/container.xml") + containerdom = parseString(container) + rootfilenodelist = containerdom.getElementsByTagName("rootfile") + rootfilename = rootfilenodelist[0].getAttribute("full-path") + + contentdom = parseString(epub.read(rootfilename)) + #firstmetadom = contentdom.getElementsByTagName("metadata")[0] + + ## Save the path to the .opf file--hrefs inside it are relative to it. + relpath = get_path_part(rootfilename) + + # spin through the manifest--only place there are item tags. + for item in contentdom.getElementsByTagName("item"): + # First, count the 'chapter' files. FFDL uses file0000.xhtml, + # but can also update epubs downloaded from Twisting the + # Hellmouth, which uses chapter0.html. + #print("---- item:%s"%item) + if( item.getAttribute("media-type") == "application/xhtml+xml" ): + filehref=relpath+item.getAttribute("href") + soup = bs.BeautifulSoup(epub.read(filehref).decode("utf-8")) + for link in soup.findAll('a',href=re.compile(r'^http.*')): + ahref=link['href'] + #print("href:(%s)"%ahref) + # hack for bad ficsaver ffnet URLs. + m = re.match(r"^http://www.fanfiction.net/s(?P<id>\d+)//$",ahref) + if m != None: + ahref="http://www.fanfiction.net/s/%s/1/"%m.group('id') + if _is_good_url == None or _is_good_url(ahref): + return ahref + return None diff --git a/fanficdownloader/exceptions.py b/fanficdownloader/exceptions.py new file mode 100644 index 0000000..2960add --- /dev/null +++ b/fanficdownloader/exceptions.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## A few exceptions for different things for adapters + +class FailedToDownload(Exception): + def __init__(self,error): + self.error=error + + def __str__(self): + return self.error + +class InvalidStoryURL(Exception): + def __init__(self,url,domain,example): + self.url=url + self.domain=domain + self.example=example + + def __str__(self): + return "Bad Story URL: (%s) for site: (%s) Example: (%s)" % (self.url, self.domain, self.example) + +class FailedToLogin(Exception): + def __init__(self,url, username, passwdonly=False): + self.url=url + self.username=username + self.passwdonly=passwdonly + + def __str__(self): + if self.passwdonly: + return "URL Failed, password required: (%s) " % (self.url) + else: + return "Failed to Login for URL: (%s) with username: (%s)" % (self.url, self.username) + +class AdultCheckRequired(Exception): + def __init__(self,url): + self.url=url + + def __str__(self): + return "Story requires confirmation of adult status: (%s)" % self.url + +class StoryDoesNotExist(Exception): + def __init__(self,url): + self.url=url + + def __str__(self): + return "Story does not exist: (%s)" % self.url + +class UnknownSite(Exception): + def __init__(self,url,supported_sites_list): + self.url=url + self.supported_sites_list=supported_sites_list + self.supported_sites_list.sort() + + def __str__(self): + return "Unknown Site(%s). Supported sites: (%s)" % (self.url, ", ".join(self.supported_sites_list)) + +class FailedToWriteOutput(Exception): + def __init__(self,error): + self.error=error + + def __str__(self): + return self.error + +class RegularExpresssionFailed(Exception): + def __init__(self,error,regex,line): + self.error=error + self.regex=regex + self.line=line + + def __str__(self): + return "Regular Expression Error '%s' in regex '%s' in line '%s'"%(self.error,self.regex,self.line) + diff --git a/fanficdownloader/geturls.py b/fanficdownloader/geturls.py new file mode 100644 index 0000000..b66202d --- /dev/null +++ b/fanficdownloader/geturls.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- + +# Copyright 2012 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import urlparse +import urllib2 as u2 + +from BeautifulSoup import BeautifulSoup +from gziphttp import GZipProcessor + +import adapters +from configurable import Configuration +from exceptions import UnknownSite + +def get_urls_from_page(url,configuration=None,normalize=False): + + if not configuration: + configuration = Configuration("test1.com","EPUB") + + data = None + adapter = None + try: + adapter = adapters.getAdapter(configuration,url,anyurl=True) + + # special stuff to log into archiveofourown.org, if possible. + # Unlike most that show the links to 'adult' stories, but protect + # them, AO3 doesn't even show them if not logged in. Only works + # with saved user/pass--not going to prompt for list. + if 'archiveofourown.org' in url: + if adapter.getConfig("username"): + if adapter.getConfig("is_adult"): + if '?' in url: + addurl = "&view_adult=true" + else: + addurl = "?view_adult=true" + else: + addurl="" + # just to get an authenticity_token. + data = adapter._fetchUrl(url+addurl) + # login the session. + adapter.performLogin(url,data) + # get the list page with logged in session. + + # this way it uses User-Agent or other special settings. Only AO3 + # is doing login. + data = adapter._fetchUrl(url,usecache=False) + except UnknownSite: + # no adapter with anyurl=True, must be a random site. + opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor()) + data = opener.open(url).read() + + # kludge because I don't see it on enough sites to be worth generalizing yet. + restrictsearch=None + if 'scarvesandcoffee.net' in url: + restrictsearch=('div',{'id':'mainpage'}) + + return get_urls_from_html(data,url,configuration,normalize,restrictsearch) + +def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None): + + normalized = [] # normalized url + retlist = [] # orig urls. + + if not configuration: + configuration = Configuration("test1.com","EPUB") + + soup = BeautifulSoup(data) + if restrictsearch: + soup = soup.find(*restrictsearch) + #print("restrict search:%s"%soup) + + for a in soup.findAll('a'): + if a.has_key('href'): + #print("a['href']:%s"%a['href']) + href = form_url(url,a['href']) + #print("1 urlhref:%s"%href) + # this (should) catch normal story links, some javascript + # 'are you old enough' links, and 'Report This' links. + # The 'normalized' set prevents duplicates. + if 'story.php' in a['href']: + #print("trying:%s"%a['href']) + m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href']) + if m != None: + href = form_url(a['href'] if '//' in a['href'] else url, + m.group('sid')) + + try: + href = href.replace('&index=1','') + #print("2 urlhref:%s"%href) + adapter = adapters.getAdapter(configuration,href) + #print("found adapter") + if adapter.story.getMetadata('storyUrl') not in normalized: + normalized.append(adapter.story.getMetadata('storyUrl')) + retlist.append(href) + except Exception, e: + #print e + pass + + if normalize: + return normalized + else: + return retlist + +def get_urls_from_text(data,configuration=None,normalize=False): + + normalized = [] # normalized url + retlist = [] # orig urls. + data=unicode(data) + + if not configuration: + configuration = Configuration("test1.com","EPUB") + + for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data): + # this (should) catch normal story links, some javascript + # 'are you old enough' links, and 'Report This' links. + # The 'normalized' set prevents duplicates. + if 'story.php' in href: + m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",href) + if m != None: + href = form_url(href,m.group('sid')) + try: + href = href.replace('&index=1','') + adapter = adapters.getAdapter(configuration,href) + if adapter.story.getMetadata('storyUrl') not in normalized: + normalized.append(adapter.story.getMetadata('storyUrl')) + retlist.append(href) + except: + pass + + if normalize: + return normalized + else: + return retlist + +def form_url(parenturl,url): + url = url.strip() # ran across an image with a space in the + # src. Browser handled it, so we'd better, too. + + if "//" in url or parenturl == None: + returl = url + else: + parsedUrl = urlparse.urlparse(parenturl) + if url.startswith("/") : + returl = urlparse.urlunparse( + (parsedUrl.scheme, + parsedUrl.netloc, + url, + '','','')) + else: + toppath="" + if parsedUrl.path.endswith("/"): + toppath = parsedUrl.path + else: + toppath = parsedUrl.path[:parsedUrl.path.rindex('/')] + returl = urlparse.urlunparse( + (parsedUrl.scheme, + parsedUrl.netloc, + toppath + '/' + url, + '','','')) + return returl + diff --git a/fanficdownloader/gziphttp.py b/fanficdownloader/gziphttp.py new file mode 100644 index 0000000..76049ee --- /dev/null +++ b/fanficdownloader/gziphttp.py @@ -0,0 +1,38 @@ +## Borrowed from http://techknack.net/python-urllib2-handlers/ + +import urllib2 +from gzip import GzipFile +from StringIO import StringIO + +class GZipProcessor(urllib2.BaseHandler): + """A handler to add gzip capabilities to urllib2 requests + """ + def http_request(self, req): + req.add_header("Accept-Encoding", "gzip") + return req + https_request = http_request + + def http_response(self, req, resp): + #print("Content-Encoding:%s"%resp.headers.get("Content-Encoding")) + if resp.headers.get("Content-Encoding") == "gzip": + gz = GzipFile( + fileobj=StringIO(resp.read()), + mode="r" + ) +# resp.read = gz.read +# resp.readlines = gz.readlines +# resp.readline = gz.readline +# resp.next = gz.next + old_resp = resp + resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) + resp.msg = old_resp.msg + return resp + https_response = http_response + +# brave new world - 1:30 w/o, 1:10 with? 40 chapters, so 20s from sleeps. +# with gzip, no sleep: 47.469 +# w/o gzip, no sleep: 47.736 + +# I Am What I Am 67 chapters +# w/o gzip: 57.168 +# w/ gzip: 40.692 diff --git a/fanficdownloader/html.py b/fanficdownloader/html.py new file mode 100644 index 0000000..22fb40a --- /dev/null +++ b/fanficdownloader/html.py @@ -0,0 +1,126 @@ +#!/usr/bin/python +# Copyright(c) 2009 Andrew Chatham and Vijay Pandurangan + +import re +import sys +import StringIO +import urllib + +from BeautifulSoup import BeautifulSoup + +class HtmlProcessor: + WHITESPACE_RE = re.compile(r'\s') + # Look for </blockquote <p> + BAD_TAG_RE = re.compile(r'<[^>]+<', re.MULTILINE) + + def __init__(self, html, unfill=0): + self.unfill = unfill + html = self._ProcessRawHtml(html) + self._soup = BeautifulSoup(html) + if self._soup.title.contents: + self.title = self._soup.title.contents[0] + else: + self.title = None + + def _ProcessRawHtml(self, html): + new_html, count = HtmlProcessor.BAD_TAG_RE.subn('<', html) + if count: + print >>sys.stderr, 'Replaced %d bad tags' % count + return new_html + + def _StubInternalAnchors(self): + '''Replace each internal anchor with a fixed-size filepos anchor. + + Looks for every anchor with <a href="#myanchor"> and replaces that + with <a filepos="00000000050">. Stores anchors in self._anchor_references''' + self._anchor_references = [] + anchor_num = 0 + # anchor links + anchorlist = self._soup.findAll('a', href=re.compile('^#')) + # treat reference tags like a tags for TOCTOP. + anchorlist.extend(self._soup.findAll('reference', href=re.compile('^#'))) + for anchor in anchorlist: + self._anchor_references.append((anchor_num, anchor['href'])) + del anchor['href'] + anchor['filepos'] = '%.10d' % anchor_num + anchor_num += 1 + + def _ReplaceAnchorStubs(self): + # TODO: Browsers allow extra whitespace in the href names. + # use __str__ instead of prettify--it inserts extra spaces. + assembled_text = self._soup.__str__('utf8') + del self._soup # shouldn't touch this anymore + for anchor_num, original_ref in self._anchor_references: + ref = urllib.unquote(original_ref[1:]) # remove leading '#' + # Find the position of ref in the utf-8 document. + # TODO(chatham): Using regexes and looking for name= would be better. + newpos = assembled_text.rfind(ref.encode('utf-8')) + if newpos == -1: + print >>sys.stderr, 'Could not find anchor "%s"' % original_ref + continue + newpos += len(ref) + 2 # don't point into the middle of the <a name> tag + old_filepos = 'filepos="%.10d"' % anchor_num + new_filepos = 'filepos="%.10d"' % newpos + assert assembled_text.find(old_filepos) != -1 + assembled_text = assembled_text.replace(old_filepos, new_filepos, 1) + return assembled_text + + def _FixPreTags(self): + '''Replace <pre> tags with HTML-ified text.''' + pres = self._soup.findAll('pre') + for pre in pres: + pre.replaceWith(self._FixPreContents(str(pre.contents[0]))) + + def _FixPreContents(self, text): + if self.unfill: + line_splitter = '\n\n' + line_joiner = '<p>' + else: + line_splitter = '\n' + line_joiner = '<br>' + lines = [] + for line in text.split(line_splitter): + lines.append(self.WHITESPACE_RE.subn(' ', line)[0]) + return line_joiner.join(lines) + + def _RemoveUnsupported(self): + '''Remove any tags which the kindle cannot handle.''' + # TODO(chatham): <link> tags to script? + unsupported_tags = ('script', 'style') + for tag_type in unsupported_tags: + for element in self._soup.findAll(tag_type): + element.extract() + + def RenameAnchors(self, prefix): + '''Rename every internal anchor to have the given prefix, then + return the contents of the body tag.''' + for anchor in self._soup.findAll('a', href=re.compile('^#')): + anchor['href'] = '#' + prefix + anchor['href'][1:] + for a in self._soup.findAll('a'): + if a.get('name'): + a['name'] = prefix + a['name'] + + # TODO(chatham): figure out how to fix this. sometimes body comes out + # as NoneType. + content = [] + if self._soup.body is not None: + content = [unicode(c) for c in self._soup.body.contents] + return '\n'.join(content) + + def CleanHtml(self): + # TODO(chatham): fix_html_br, fix_html + self._RemoveUnsupported() + self._StubInternalAnchors() + self._FixPreTags() + return self._ReplaceAnchorStubs() + + +if __name__ == '__main__': + FILE ='/tmp/documentation.html' + #FILE = '/tmp/multipre.html' + FILE = '/tmp/view.html' + import codecs + d = open(FILE).read() + h = HtmlProcessor(d) + s = h.CleanHtml() + #print s diff --git a/fanficdownloader/html2text.py b/fanficdownloader/html2text.py new file mode 100644 index 0000000..1996527 --- /dev/null +++ b/fanficdownloader/html2text.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""html2text: Turn HTML into equivalent Markdown-structured text.""" +__version__ = "2.37" +__author__ = "Aaron Swartz (me@aaronsw.com)" +__copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3." +__contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"] + +# TODO: +# Support decoded entities with unifiable. + +if not hasattr(__builtins__, 'True'): True, False = 1, 0 +import re, sys, urllib, htmlentitydefs, codecs, StringIO, types +import sgmllib +import urlparse +sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]') + +try: from textwrap import wrap +except: pass + +# Use Unicode characters instead of their ascii psuedo-replacements +UNICODE_SNOB = 0 + +# Put the links after each paragraph instead of at the end. +LINKS_EACH_PARAGRAPH = 0 + +# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.) +BODY_WIDTH = 78 + +# Don't show internal links (href="#local-anchor") -- corresponding link targets +# won't be visible in the plain text file anyway. +SKIP_INTERNAL_LINKS = False + +### Entity Nonsense ### + +def name2cp(k): + if k == 'apos': return ord("'") + if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3 + return htmlentitydefs.name2codepoint[k] + else: + k = htmlentitydefs.entitydefs[k] + if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1 + return ord(codecs.latin_1_decode(k)[0]) + +unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"', +'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*', +'ndash':'-', 'oelig':'oe', 'aelig':'ae', +'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a', +'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e', +'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i', +'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o', +'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u'} + +unifiable_n = {} + +for k in unifiable.keys(): + unifiable_n[name2cp(k)] = unifiable[k] + +def charref(name): + if name[0] in ['x','X']: + c = int(name[1:], 16) + else: + c = int(name) + + if not UNICODE_SNOB and c in unifiable_n.keys(): + return unifiable_n[c] + else: + return unichr(c) + +def entityref(c): + if not UNICODE_SNOB and c in unifiable.keys(): + return unifiable[c] + else: + try: name2cp(c) + except KeyError: return "&" + c + else: return unichr(name2cp(c)) + +def replaceEntities(s): + s = s.group(1) + if s[0] == "#": + return charref(s[1:]) + else: return entityref(s) + +r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));") +def unescape(s): + return r_unescape.sub(replaceEntities, s) + +def fixattrs(attrs): + # Fix bug in sgmllib.py + if not attrs: return attrs + newattrs = [] + for attr in attrs: + newattrs.append((attr[0], unescape(attr[1]))) + return newattrs + +### End Entity Nonsense ### + +def onlywhite(line): + """Return true if the line does only consist of whitespace characters.""" + for c in line: + if c is not ' ' and c is not ' ': + return c is ' ' + return line + +def optwrap(text,wrap_width=BODY_WIDTH): + """Wrap all paragraphs in the provided text.""" + + if not wrap_width: + return text + + assert wrap, "Requires Python 2.3." + result = '' + newlines = 0 + for para in text.split("\n"): + if len(para) > 0: + if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*': + for line in wrap(para, wrap_width): + result += line + "\n" + result += "\n" + newlines = 2 + else: + if not onlywhite(para): + result += para + "\n" + newlines = 1 + else: + if newlines < 2: + result += "\n" + newlines += 1 + return result + +def hn(tag): + if tag[0] == 'h' and len(tag) == 2: + try: + n = int(tag[1]) + if n in range(1, 10): return n + except ValueError: return 0 + +class _html2text(sgmllib.SGMLParser): + def __init__(self, out=None, baseurl=''): + sgmllib.SGMLParser.__init__(self) + + if out is None: self.out = self.outtextf + else: self.out = out + self.outtext = u'' + self.quiet = 0 + self.p_p = 0 + self.outcount = 0 + self.start = 1 + self.space = 0 + self.a = [] + self.astack = [] + self.acount = 0 + self.list = [] + self.blockquote = 0 + self.pre = 0 + self.startpre = 0 + self.lastWasNL = 0 + self.abbr_title = None # current abbreviation definition + self.abbr_data = None # last inner HTML (for abbr being defined) + self.abbr_list = {} # stack of abbreviations to write later + self.baseurl = baseurl + + def outtextf(self, s): + self.outtext += s + + def close(self): + sgmllib.SGMLParser.close(self) + + self.pbr() + self.o('', 0, 'end') + + return self.outtext + + def handle_charref(self, c): + self.o(charref(c)) + + def handle_entityref(self, c): + self.o(entityref(c)) + + def unknown_starttag(self, tag, attrs): + self.handle_tag(tag, attrs, 1) + + def unknown_endtag(self, tag): + self.handle_tag(tag, None, 0) + + def previousIndex(self, attrs): + """ returns the index of certain set of attributes (of a link) in the + self.a list + + If the set of attributes is not found, returns None + """ + if not attrs.has_key('href'): return None + + i = -1 + for a in self.a: + i += 1 + match = 0 + + if a.has_key('href') and a['href'] == attrs['href']: + if a.has_key('title') or attrs.has_key('title'): + if (a.has_key('title') and attrs.has_key('title') and + a['title'] == attrs['title']): + match = True + else: + match = True + + if match: return i + + def handle_tag(self, tag, attrs, start): + attrs = fixattrs(attrs) + + if hn(tag): + self.p() + if start: self.o(hn(tag)*"#" + ' ') + + if tag in ['p', 'div']: self.p() + + if tag == "br" and start: self.o(" \n") + + if tag == "hr" and start: + self.p() + self.o("* * *") + self.p() + + if tag in ["head", "style", 'script']: + if start: self.quiet += 1 + else: self.quiet -= 1 + + if tag in ["body"]: + self.quiet = 0 # sites like 9rules.com never close <head> + + if tag == "blockquote": + if start: + self.p(); self.o('> ', 0, 1); self.start = 1 + self.blockquote += 1 + else: + self.blockquote -= 1 + self.p() + + if tag in ['em', 'i', 'u']: self.o("_") + if tag in ['strong', 'b']: self.o("**") + if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` `` + if tag == "abbr": + if start: + attrsD = {} + for (x, y) in attrs: attrsD[x] = y + attrs = attrsD + + self.abbr_title = None + self.abbr_data = '' + if attrs.has_key('title'): + self.abbr_title = attrs['title'] + else: + if self.abbr_title != None: + self.abbr_list[self.abbr_data] = self.abbr_title + self.abbr_title = None + self.abbr_data = '' + + if tag == "a": + if start: + attrsD = {} + for (x, y) in attrs: attrsD[x] = y + attrs = attrsD + if attrs.has_key('href') and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')): + self.astack.append(attrs) + self.o("[") + else: + self.astack.append(None) + else: + if self.astack: + a = self.astack.pop() + if a: + i = self.previousIndex(a) + if i is not None: + a = self.a[i] + else: + self.acount += 1 + a['count'] = self.acount + a['outcount'] = self.outcount + self.a.append(a) + self.o("][" + `a['count']` + "]") + + if tag == "img" and start: + attrsD = {} + for (x, y) in attrs: attrsD[x] = y + attrs = attrsD + if attrs.has_key('src'): + attrs['href'] = attrs['src'] + alt = attrs.get('alt', '') + i = self.previousIndex(attrs) + if i is not None: + attrs = self.a[i] + else: + self.acount += 1 + attrs['count'] = self.acount + attrs['outcount'] = self.outcount + self.a.append(attrs) + self.o("![") + self.o(alt) + self.o("]["+`attrs['count']`+"]") + + if tag == 'dl' and start: self.p() + if tag == 'dt' and not start: self.pbr() + if tag == 'dd' and start: self.o(' ') + if tag == 'dd' and not start: self.pbr() + + if tag in ["ol", "ul"]: + if start: + self.list.append({'name':tag, 'num':0}) + else: + if self.list: self.list.pop() + + self.p() + + if tag == 'li': + if start: + self.pbr() + if self.list: li = self.list[-1] + else: li = {'name':'ul', 'num':0} + self.o(" "*len(self.list)) #TODO: line up <ol><li>s > 9 correctly. + if li['name'] == "ul": self.o("* ") + elif li['name'] == "ol": + li['num'] += 1 + self.o(`li['num']`+". ") + self.start = 1 + else: + self.pbr() + + if tag in ["table", "tr"] and start: self.p() + if tag == 'td': self.pbr() + + if tag == "pre": + if start: + self.startpre = 1 + self.pre = 1 + else: + self.pre = 0 + self.p() + + def pbr(self): + if self.p_p == 0: self.p_p = 1 + + def p(self): self.p_p = 2 + + def o(self, data, puredata=0, force=0): + if self.abbr_data is not None: self.abbr_data += data + + if not self.quiet: + if puredata and not self.pre: + data = re.sub('\s+', ' ', data) + if data and data[0] == ' ': + self.space = 1 + data = data[1:] + if not data and not force: return + + if self.startpre: + #self.out(" :") #TODO: not output when already one there + self.startpre = 0 + + bq = (">" * self.blockquote) + if not (force and data and data[0] == ">") and self.blockquote: bq += " " + + if self.pre: + bq += " " + data = data.replace("\n", "\n"+bq) + + if self.start: + self.space = 0 + self.p_p = 0 + self.start = 0 + + if force == 'end': + # It's the end. + self.p_p = 0 + self.out("\n") + self.space = 0 + + + if self.p_p: + self.out(('\n'+bq)*self.p_p) + self.space = 0 + + if self.space: + if not self.lastWasNL: self.out(' ') + self.space = 0 + + if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"): + if force == "end": self.out("\n") + + newa = [] + for link in self.a: + if self.outcount > link['outcount']: + self.out(" ["+`link['count']`+"]: " + urlparse.urljoin(self.baseurl, link['href'])) + if link.has_key('title'): self.out(" ("+link['title']+")") + self.out("\n") + else: + newa.append(link) + + if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done. + + self.a = newa + + if self.abbr_list and force == "end": + for abbr, definition in self.abbr_list.items(): + self.out(" *[" + abbr + "]: " + definition + "\n") + + self.p_p = 0 + self.out(data) + self.lastWasNL = data and data[-1] == '\n' + self.outcount += 1 + + def handle_data(self, data): + if r'\/script>' in data: self.quiet -= 1 + self.o(data, 1) + + def unknown_decl(self, data): pass + +def wrapwrite(text): sys.stdout.write(text.encode('utf8')) + +def html2text_file(html, out=wrapwrite, baseurl=''): + h = _html2text(out, baseurl) + h.feed(html) + h.feed("") + return h.close() + +def html2text(html, baseurl='', wrap_width=BODY_WIDTH): + return optwrap(html2text_file(html, None, baseurl),wrap_width) + +if __name__ == "__main__": + baseurl = '' + if sys.argv[1:]: + arg = sys.argv[1] + if arg.startswith('http://'): + baseurl = arg + j = urllib.urlopen(baseurl) + try: + from feedparser import _getCharacterEncoding as enc + except ImportError: + enc = lambda x, y: ('utf-8', 1) + text = j.read() + encoding = enc(j.headers, text)[0] + if encoding == 'us-ascii': encoding = 'utf-8' + data = text.decode(encoding) + + else: + encoding = 'utf8' + if len(sys.argv) > 2: + encoding = sys.argv[2] + data = open(arg, 'r').read().decode(encoding) + else: + data = sys.stdin.read().decode('utf8') + wrapwrite(html2text(data, baseurl)) diff --git a/fanficdownloader/htmlcleanup.py b/fanficdownloader/htmlcleanup.py new file mode 100644 index 0000000..325f5e1 --- /dev/null +++ b/fanficdownloader/htmlcleanup.py @@ -0,0 +1,485 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +logger = logging.getLogger(__name__) + +import re + +def _unirepl(match): + "Return the unicode string for a decimal number" + if match.group(1).startswith('x'): + radix=16 + s = match.group(1)[1:] + else: + radix=10 + s = match.group(1) + try: + value = int(s, radix) + retval = "%s%s"%(unichr(value),match.group(2)) + except: + # This way, at least if there's more of entities out there + # that fail, it doesn't blow the entire download. + logger.warn("Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2))) + retval = "" + return retval + +def _replaceNumberEntities(data): + # The same brokenish entity parsing in SGMLParser that inserts ';' + # after non-entities will also insert ';' incorrectly after number + # entities, including part of the next word if it's a-z. + # "Don't—ever—do—that—again," becomes + # "Don't—e;ver—d;o—that—a;gain," + # Also need to allow for 5 digit decimal entities 法 + # Last expression didn't allow for 2 digit hex correctly: é + p = re.compile(r'&#(x[0-9a-fA-F]{,4}|[0-9]{,5})([0-9a-fA-F]*?);') + return p.sub(_unirepl, data) + +def _replaceNotEntities(data): + # not just \w or \S. regexp from c:\Python25\lib\sgmllib.py + # (or equiv), SGMLParser, entityref + p = re.compile(r'&([a-zA-Z][-.a-zA-Z0-9]*);') + return p.sub(r'&\1', data) + +def stripHTML(soup): + if isinstance(soup,basestring) or hasattr(soup, 'bs3'): + return removeAllEntities(re.sub(r'<[^>]+>','',"%s" % soup)).strip() + else: + # bs4 already converts all the entities to UTF8 chars. + return soup.get_text(strip=True) + +def conditionalRemoveEntities(value): + if isinstance(value,basestring): + return removeEntities(value).strip() + else: + return value + +def removeAllEntities(text): + # Remove < < and & + return removeEntities(text).replace('<', '<').replace('>', '>').replace('&', '&') + +def removeEntities(text): + if text is None: + return u"" + + if not isinstance(text,basestring): + return unicode(text) + + try: + t = text.decode('utf-8') + except UnicodeEncodeError, e: + try: + t = text.encode ('ascii', 'xmlcharrefreplace') + except UnicodeEncodeError, e: + t = text + text = t + # replace numeric versions of [&<>] with named versions, + # then replace named versions with actual characters, + text = re.sub(r'�*38;','&',text) + text = re.sub(r'�*60;','<',text) + text = re.sub(r'�*62;','>',text) + + # replace remaining � entities with unicode value, such as ' -> ' + text = _replaceNumberEntities(text) + + # replace several named entities with character, such as — -> - + # see constants.py for the list. + # reverse sort will put entities with ; before the same one without, when valid. + for e in reversed(sorted(entities.keys())): + v = entities[e] + try: + text = text.replace(e, v) + except UnicodeDecodeError, ex: + # for the pound symbol in constants.py + text = text.replace(e, v.decode('utf-8')) + + # SGMLParser, and in turn, BeautifulStoneSoup doesn't parse + # entities terribly well and inserts (;) after something that + # it thinks might be an entity. AT&T becomes AT&T; All of my + # attempts to fix this by changing the input to + # BeautifulStoneSoup break something else instead. But at + # this point, there should be *no* real entities left, so find + # these not-entities and removing them here should be safe. + text = _replaceNotEntities(text) + + # < < and & are the only html entities allowed in xhtml, put those back. + return text.replace('&', '&').replace('&lt', '<').replace('&gt', '>') + +# entity list from http://code.google.com/p/doctype/wiki/CharacterEntitiesConsistent +entities = { 'á' : 'á', + 'Á' : 'Á', + 'Á' : 'Á', + 'á' : 'á', + 'â' : 'â', + 'Â' : 'Â', + 'Â' : 'Â', + 'â' : 'â', + '´' : '´', + '´' : '´', + 'Æ' : 'Æ', + 'æ' : 'æ', + 'Æ' : 'Æ', + 'æ' : 'æ', + 'à' : 'à', + 'À' : 'À', + 'À' : 'À', + 'à' : 'à', + 'ℵ' : 'ℵ', + 'α' : 'α', + 'Α' : 'Α', + '&' : '&', + '&' : '&', + '&' : '&', + '&' : '&', + '∧' : '∧', + '∠' : '∠', + 'å' : 'å', + 'Å' : 'Å', + 'Å' : 'Å', + 'å' : 'å', + '≈' : '≈', + 'ã' : 'ã', + 'Ã' : 'Ã', + 'Ã' : 'Ã', + 'ã' : 'ã', + 'ä' : 'ä', + 'Ä' : 'Ä', + 'Ä' : 'Ä', + 'ä' : 'ä', + '„' : '„', + 'β' : 'β', + 'Β' : 'Β', + '¦' : '¦', + '¦' : '¦', + '•' : '•', + '∩' : '∩', + 'ç' : 'ç', + 'Ç' : 'Ç', + 'Ç' : 'Ç', + 'ç' : 'ç', + '¸' : '¸', + '¸' : '¸', + '¢' : '¢', + '¢' : '¢', + 'χ' : 'χ', + 'Χ' : 'Χ', + 'ˆ' : 'ˆ', + '♣' : '♣', + '≅' : '≅', + '©' : '©', + '©' : '©', + '©' : '©', + '©' : '©', + '↵' : '↵', + '∪' : '∪', + '¤' : '¤', + '¤' : '¤', + '†' : '†', + '‡' : '‡', + '↓' : '↓', + '⇓' : '⇓', + '°' : '°', + '°' : '°', + 'δ' : 'δ', + 'Δ' : 'Δ', + '♦' : '♦', + '÷' : '÷', + '÷' : '÷', + 'é' : 'é', + 'É' : 'É', + 'É' : 'É', + 'é' : 'é', + 'ê' : 'ê', + 'Ê' : 'Ê', + 'Ê' : 'Ê', + 'ê' : 'ê', + 'è' : 'è', + 'È' : 'È', + 'È' : 'È', + 'è' : 'è', + '∅' : '∅', + ' ' : ' ', + ' ' : ' ', + 'ε' : 'ε', + 'Ε' : 'Ε', + '≡' : '≡', + 'η' : 'η', + 'Η' : 'Η', + 'ð' : 'ð', + 'Ð' : 'Ð', + 'Ð' : 'Ð', + 'ð' : 'ð', + 'ë' : 'ë', + 'Ë' : 'Ë', + 'Ë' : 'Ë', + 'ë' : 'ë', + '€' : '€', + '∃' : '∃', + 'ƒ' : 'ƒ', + '∀' : '∀', + '½' : '½', + '½' : '½', + '¼' : '¼', + '¼' : '¼', + '¾' : '¾', + '¾' : '¾', + '⁄' : '⁄', + 'γ' : 'γ', + 'Γ' : 'Γ', + '≥' : '≥', + #'>' : '>', + #'>' : '>', + #'>' : '>', + #'>' : '>', + '↔' : '↔', + '⇔' : '⇔', + '♥' : '♥', + '…' : '…', + 'í' : 'í', + 'Í' : 'Í', + 'Í' : 'Í', + 'í' : 'í', + 'î' : 'î', + 'Î' : 'Î', + 'Î' : 'Î', + 'î' : 'î', + '¡' : '¡', + '¡' : '¡', + 'ì' : 'ì', + 'Ì' : 'Ì', + 'Ì' : 'Ì', + 'ì' : 'ì', + 'ℑ' : 'ℑ', + '∞' : '∞', + '∫' : '∫', + 'ι' : 'ι', + 'Ι' : 'Ι', + '¿' : '¿', + '¿' : '¿', + '∈' : '∈', + 'ï' : 'ï', + 'Ï' : 'Ï', + 'Ï' : 'Ï', + 'ï' : 'ï', + 'κ' : 'κ', + 'Κ' : 'Κ', + 'λ' : 'λ', + 'Λ' : 'Λ', + '«' : '«', + '«' : '«', + '←' : '←', + '⇐' : '⇐', + '⌈' : '⌈', + '“' : '“', + '≤' : '≤', + '⌊' : '⌊', + '∗' : '∗', + '◊' : '◊', + '‎' : '‎', + '‹' : '‹', + '‘' : '‘', + #'<' : '<', + #'<' : '<', + #'<' : '<', + #'<' : '<', + '¯' : '¯', + '¯' : '¯', + '—' : '—', + 'µ' : 'µ', + 'µ' : 'µ', + '·' : '·', + '·' : '·', + '−' : '−', + 'μ' : 'μ', + 'Μ' : 'Μ', + '∇' : '∇', + ' ' : ' ', + ' ' : ' ', + '–' : '–', + '≠' : '≠', + '∋' : '∋', + '¬' : '¬', + '¬' : '¬', + '∉' : '∉', + '⊄' : '⊄', + 'ñ' : 'ñ', + 'Ñ' : 'Ñ', + 'Ñ' : 'Ñ', + 'ñ' : 'ñ', + 'ν' : 'ν', + 'Ν' : 'Ν', + 'ó' : 'ó', + 'Ó' : 'Ó', + 'Ó' : 'Ó', + 'ó' : 'ó', + 'ô' : 'ô', + 'Ô' : 'Ô', + 'Ô' : 'Ô', + 'ô' : 'ô', + 'Œ' : 'Œ', + 'œ' : 'œ', + 'ò' : 'ò', + 'Ò' : 'Ò', + 'Ò' : 'Ò', + 'ò' : 'ò', + '‾' : '‾', + 'ω' : 'ω', + 'Ω' : 'Ω', + 'ο' : 'ο', + 'Ο' : 'Ο', + '⊕' : '⊕', + '∨' : '∨', + 'ª' : 'ª', + 'ª' : 'ª', + 'º' : 'º', + 'º' : 'º', + 'ø' : 'ø', + 'Ø' : 'Ø', + 'Ø' : 'Ø', + 'ø' : 'ø', + 'õ' : 'õ', + 'Õ' : 'Õ', + 'Õ' : 'Õ', + 'õ' : 'õ', + '⊗' : '⊗', + 'ö' : 'ö', + 'Ö' : 'Ö', + 'Ö' : 'Ö', + 'ö' : 'ö', + '¶' : '¶', + '¶' : '¶', + '∂' : '∂', + '‰' : '‰', + '⊥' : '⊥', + 'φ' : 'φ', + 'Φ' : 'Φ', + 'π' : 'π', + 'Π' : 'Π', + 'ϖ' : 'ϖ', + '±' : '±', + '±' : '±', + '£' : '£', + '£' : '£', + '′' : '′', + '″' : '″', + '∏' : '∏', + '∝' : '∝', + 'ψ' : 'ψ', + 'Ψ' : 'Ψ', + '"' : '"', + '"' : '"', + '"' : '"', + '"' : '"', + '√' : '√', + '»' : '»', + '»' : '»', + '→' : '→', + '⇒' : '⇒', + '⌉' : '⌉', + '”' : '”', + 'ℜ' : 'ℜ', + '®' : '®', + '®' : '®', + '®' : '®', + '®' : '®', + '⌋' : '⌋', + 'ρ' : 'ρ', + 'Ρ' : 'Ρ', + '‏' : '‏', + '›' : '›', + '’' : '’', + '‚' : '‚', + 'š' : 'š', + 'Š' : 'Š', + '⋅' : '⋅', + '§' : '§', + '§' : '§', + '­' : '­', # strange optional hyphenation control character, not just a dash + '­' : '­', + 'σ' : 'σ', + 'Σ' : 'Σ', + 'ς' : 'ς', + '∼' : '∼', + '♠' : '♠', + '⊂' : '⊂', + '⊆' : '⊆', + '∑' : '∑', + '¹' : '¹', + '¹' : '¹', + '²' : '²', + '²' : '²', + '³' : '³', + '³' : '³', + '⊃' : '⊃', + '⊇' : '⊇', + 'ß' : 'ß', + 'ß' : 'ß', + 'τ' : 'τ', + 'Τ' : 'Τ', + '∴' : '∴', + 'θ' : 'θ', + 'Θ' : 'Θ', + 'ϑ' : 'ϑ', + ' ' : ' ', + 'þ' : 'þ', + 'Þ' : 'Þ', + 'Þ' : 'Þ', + 'þ' : 'þ', + '˜' : '˜', + '×' : '×', + '×' : '×', + '™' : '™', + 'ú' : 'ú', + 'Ú' : 'Ú', + 'Ú' : 'Ú', + 'ú' : 'ú', + '↑' : '↑', + '⇑' : '⇑', + 'û' : 'û', + 'Û' : 'Û', + 'Û' : 'Û', + 'û' : 'û', + 'ù' : 'ù', + 'Ù' : 'Ù', + 'Ù' : 'Ù', + 'ù' : 'ù', + '¨' : '¨', + '¨' : '¨', + 'ϒ' : 'ϒ', + 'υ' : 'υ', + 'Υ' : 'Υ', + 'ü' : 'ü', + 'Ü' : 'Ü', + 'Ü' : 'Ü', + 'ü' : 'ü', + '℘' : '℘', + 'ξ' : 'ξ', + 'Ξ' : 'Ξ', + 'ý' : 'ý', + 'Ý' : 'Ý', + 'Ý' : 'Ý', + 'ý' : 'ý', + '¥' : '¥', + '¥' : '¥', + 'ÿ' : 'ÿ', + 'Ÿ' : 'Ÿ', + 'ÿ' : 'ÿ', + 'ζ' : 'ζ', + 'Ζ' : 'Ζ', + '‍' : '‍', # strange spacing control character, not just a space + '‌' : '‌', # strange spacing control character, not just a space + } diff --git a/fanficdownloader/htmlheuristics.py b/fanficdownloader/htmlheuristics.py new file mode 100644 index 0000000..52a78df --- /dev/null +++ b/fanficdownloader/htmlheuristics.py @@ -0,0 +1,348 @@ +# -*- coding: utf-8 -*- + +# Copyright 2013 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +logger = logging.getLogger(__name__) +import re +import codecs +import BeautifulSoup as bs +import HtmlTagStack as stack + +from . import exceptions as exceptions + +def replace_br_with_p(body): + + # Ascii character (and Unicode as well) xA0 is a non-breaking space, ascii code 160. + # However, Python Regex does not recognize it as a whitespace, so we'll be changing it to a reagular space. + body = body.replace(u'\xa0', u' ') + + if body.find('>') == -1 or body.rfind('<') == -1: + return body + + # logger.debug(u'BODY start.: ' + body[:250]) + # logger.debug(u'BODY end...: ' + body[-250:]) + # logger.debug(u'BODY.......: ' + body) + + # clean breaks (<br />), removing whitespaces between them. + body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body) + + # change surrounding div to a p and remove attrs Top surrounding + # tag in all cases now should be div, to just strip the first and + # last tags. + if is_valid_block(body) and body.find('<div') == 0: + body = body[body.index('>')+1:body.rindex('<')] + + body = soup_up_div(u'<div>' + body + u'</div>') + + body = body[body.index('>')+1:body.rindex('<')] + + # Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those. + # This is for "lenient" mode, however it is also used to clear break tags before and after the block elements. + blocksRegex = re.compile(r'(\s*<br\ />\s*)*\s*<(pre|p|blockquote|table)([^>]*)>(.+?)</\2>\s*(\s*<br\ />\s*)*', re.DOTALL) + body = blocksRegex.sub(r'\n<\2\3>\4</\2>\n', body) + + # if aggressive mode = true + # blocksRegex = re.compile(r'(\s*<br\ */*>\s*)*\s*<(pre)([^>]*)>(.+?)</\2>\s*(\s*<br\ */*>\s*)*', re.DOTALL) + # In aggressive mode, we also check breakes inside blockquotes, meaning we can get orphaned paragraph tags. + # body = re.sub(r'<blockquote([^>]*)>(.+?)</blockquote>', r'<blockquote\1><p>\2</p></blockquote>', body, re.DOTALL) + # end aggressive mode + + blocks = blocksRegex.finditer(body) + # For our replacements to work, we need to work backwards, so we reverse the iterator. + blocksList = [] + for match in blocks: + blocksList.insert(0, match) + + for match in blocksList: + group4 = match.group(4).replace(u'<br />', u'{br /}') + body = body[:match.start(4)] + group4 + body[match.end(4):] + + # change surrounding div to a p and remove attrs Top surrounding + # tag in all cases now should be div, to just strip the first and + # last tags. + # body = u'<p>' + body + u'</p>' + + # Nuke div tags surrounding a HR tag. + body = re.sub(r'<div[^>]+>\s*<hr[^>]+>\s*</div>', r'\n<hr />\n', body) + + # So many people add formatting to their HR tags, and ePub does not allow those, we are supposed to use css. + # This nukes the hr tag attributes. + body = re.sub(r'\s*<hr[^>]+>\s*', r'\n<hr />\n', body) + + # Remove leading and trailing breaks from HR tags + body = re.sub(r'\s*(<br\ \/>)*\s*<hr\ \/>\s*(<br\ \/>)*\s*', r'\n<hr />\n', body) + # Nuking breaks leading paragraps that may be in the body. They are eventually treated as <p><br /></p> + body = re.sub(r'\s*(<br\ \/>)+\s*<p', r'\n<p></p>\n<p', body) + # Nuking breaks trailing paragraps that may be in the body. They are eventually treated as <p><br /></p> + body = re.sub(r'</p>\s*(<br\ \/>)+\s*', r'</p>\n<p></p>\n', body) + + # Because a leading or trailing non break tag will break the following code, we have to mess around rather badly for a few lines. + body = body.replace(u'[',u'&squareBracketStart;') + body = body.replace(u']',u'&squareBracketEnd;') + body = body.replace(u'<br />',u'[br /]') + + breaksRegexp = [ + re.compile(r'([^\]])(\[br\ \/\])([^\[])'), + re.compile(r'([^\]])(\[br\ \/\]){2}([^\[])'), + re.compile(r'([^\]])(\[br\ \/\]){3}([^\[])'), + re.compile(r'([^\]])(\[br\ \/\]){4}([^\[])'), + re.compile(r'([^\]])(\[br\ \/\]){5}([^\[])'), + re.compile(r'([^\]])(\[br\ \/\]){6}([^\[])'), + re.compile(r'([^\]])(\[br\ \/\]){7}([^\[])'), + re.compile(r'([^\]])(\[br\ \/\]){8}([^\[])'), + re.compile(r'(\[br\ \/\]){9,}')] + + breaksCount = [ + len(breaksRegexp[0].findall(body)), + len(breaksRegexp[1].findall(body)), + len(breaksRegexp[2].findall(body)), + len(breaksRegexp[3].findall(body)), + len(breaksRegexp[4].findall(body)), + len(breaksRegexp[5].findall(body)), + len(breaksRegexp[6].findall(body)), + len(breaksRegexp[7].findall(body))] + + breaksMax = 0 + breaksMaxIndex = 0; + + for i in range(1,len(breaksCount)): + if breaksCount[i] >= breaksMax: + breaksMax = breaksCount[i] + breaksMaxIndex = i + + lines = body.split(u'[br /]') + contentLines = 0; + contentLinesSum = 0; + longestLineLength = 0; + averageLineLength = 0; + + for line in lines: + lineLen = len(line.strip()) + if lineLen > 0: + contentLines += 1 + contentLinesSum += lineLen + if lineLen > longestLineLength: + longestLineLength = lineLen + + if contentLines == 0: + contentLines = 1 + + averageLineLength = contentLinesSum/contentLines + + logger.debug(u'---') + logger.debug(u'Lines.............: ' + unicode(len(lines))) + logger.debug(u'contentLines......: ' + unicode(contentLines)) + logger.debug(u'contentLinesSum...: ' + unicode(contentLinesSum)) + logger.debug(u'longestLineLength.: ' + unicode(longestLineLength)) + logger.debug(u'averageLineLength.: ' + unicode(averageLineLength)) + + if breaksMaxIndex == len(breaksCount)-1 and breaksMax < 2: + breaksMaxIndex = 0 + breaksMax = breaksCount[0] + + logger.debug(u'---') + logger.debug(u'breaks 1: ' + unicode(breaksCount[0])) + logger.debug(u'breaks 2: ' + unicode(breaksCount[1])) + logger.debug(u'breaks 3: ' + unicode(breaksCount[2])) + logger.debug(u'breaks 4: ' + unicode(breaksCount[3])) + logger.debug(u'breaks 5: ' + unicode(breaksCount[4])) + logger.debug(u'breaks 6: ' + unicode(breaksCount[5])) + logger.debug(u'breaks 7: ' + unicode(breaksCount[6])) + logger.debug(u'breaks 8: ' + unicode(breaksCount[7])) + logger.debug(u'----') + logger.debug(u'max found: ' + unicode(breaksMax)) + logger.debug(u'max Index: ' + unicode(breaksMaxIndex)) + logger.debug(u'----') + + if breaksMaxIndex > 0 and breaksCount[0] > breaksMax and averageLineLength < 90: + body = breaksRegexp[0].sub(r'\1 \n\3', body) + + # Find all instances of consecutive breaks less than otr equal to the max count use most often + # replase those tags to inverted p tag pairs, those with more connsecutive breaks are replaced them with a horisontal line + for i in range(len(breaksCount)): + # if i > 0 or breaksMaxIndex == 0: + if i <= breaksMaxIndex: + logger.debug(unicode(i) + u' <= breaksMaxIndex (' + unicode(breaksMaxIndex) + u')') + body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body) + elif i == breaksMaxIndex+1: + logger.debug(unicode(i) + u' == breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')') + body = breaksRegexp[i].sub(r'\1</p>\n<p><br/></p>\n<p>\3', body) + else: + logger.debug(unicode(i) + u' > breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')') + body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body) + + body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body) + + # Reverting the square brackets + body = body.replace(u'[', u'<') + body = body.replace(u']', u'>') + body = body.replace(u'&squareBracketStart;', u'[') + body = body.replace(u'&squareBracketEnd;', u']') + + body = body.replace(u'{p}', u'<p>') + body = body.replace(u'{/p}', u'</p>') + + # If for some reason, a third break makes its way inside the paragraph, preplace that with the empty paragraph for the additional linespaing. + body = re.sub(r'<p>\s*(<br\ \/>)+', r'<p><br /></p>\n<p>', body) + + # change empty p tags to include a br to force spacing. + body = re.sub(r'<p>\s*</p>', r'<p><br/></p>', body) + + # Clean up hr tags, and add inverted p tag pairs + body = re.sub(r'(<div[^>]+>)*\s*<hr\ \/>\s*(</div>)*', r'\n<hr />\n', body) + + # Clean up hr tags, and add inverted p tag pairs + body = re.sub(r'\s*<hr\ \/>\s*', r'</p>\n<hr />\n<p>', body) + + # Because the previous regexp may cause trouble if the hr tag already had a p tag pair around it, w nee dot repair that. + # Repeated opening p tags are condenced to one. As we added the extra leading opening p tags, we can safely assume that + # the last in such a chain must be the original. Lets keep its attributes if they are there. + body = re.sub(r'\s*(<p[^>]*>\s*)+<p([^>]*)>\s*', r'\n<p\2>', body) + # Repeated closing p tags are condenced to one + body = re.sub(r'\s*(<\/\s*p>\s*){2,}', r'</p>\n', body) + + # superflous cleaning, remove whitespaces traling opening p tags. These does affect formatting. + body = re.sub(r'\s*<p([^>]*)>\s*', r'\n<p\1>', body) + # superflous cleaning, remove whitespaces leading closing p tags. These does not affect formatting. + body = re.sub(r'\s*</p>\s*', r'</p>\n', body) + + # Remove empty tag pairs + body = re.sub(r'\s*<(\S+)[^>]*>\s*</\1>', r'', body) + + body = body.replace(u'{br /}', u'<br />') + body = body.strip() + + # re-wrap in div tag. + body = u'<div>\n' + body + u'</div>\n' + + # return body + return tag_sanitizer(body) + +def is_valid_block(block): + return unicode(block).find('<') == 0 and unicode(block).find('<!') != 0 + +def soup_up_div(body): + blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'p', 'pre', 'table', 'ul'] + recurseTags = ['blockquote', 'div', 'noscript'] + + tag = body[:body.index('>')+1] + tagend = body[body.rindex('<'):] + + body = body.replace(u'<br />', u'[br /]') + + soup = bs.BeautifulSoup(body) + + body = u'' + lastElement = 1 # 1 = block, 2 = nested, 3 = invalid + + for i in soup.contents[0]: + if unicode(i).strip().__len__() > 0: + s = unicode(i) + if type(i) == bs.Tag: + if i.name in blockTags: + if lastElement > 1: + body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*') + body += u'{/p}' + + lastElement = 1 + + if i.name in recurseTags: + s = soup_up_div(s) + + body += s.strip() + '\n' + else: + if lastElement == 1: + body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*') + body += u'{p}' + + lastElement = 2 + body += s + elif type(i) == bs.Comment: + body += s + else: + if lastElement == 1: + body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*') + body += u'{p}' + + lastElement = 3 + body += s + + if lastElement > 1: + body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*') + body += u'{/p}' + + body = body.replace(u'[br /]', u'<br />') + + return tag + body + tagend + + +def is_end_tag(tag): + return re.match(r'</([^\ >]+)>', tag) != None + +def is_comment_tag(tag): + return re.match(r'<\!\-\-([^>]+)>', tag) != None + +def is_closed_tag(tag): + return re.match(r'<(.+?)/>', tag) != None + +def tag_sanitizer(html): + blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'pre', 'table', 'ul'] + + body = u'' + tags = re.findall(r'(<[^>]+>)([^<]*)', html) + + for rTag in tags: + name = stack.get_tag_name(rTag[0]) + is_end = is_end_tag(rTag[0]) + is_closed = is_closed_tag(rTag[0]) or is_comment_tag(rTag[0]) + + # is_comment = is_comment_tag(rTag[0]) + # logger.debug(u'%s > isEnd: %s > isClosed: %s > isComment: %s'%(name, unicode(is_end), unicode(is_closed), unicode(is_comment))) + # logger.debug(u'> %s%s\n'%(rTag[0], rTag[1])) + + if name in blockTags: + body += rTag[0] + body += rTag[1] + elif name == u'p': + if is_end: + body += stack.spool_end() + body += rTag[0] + body += rTag[1] + elif is_closed: + body += rTag[0] + body += rTag[1] + else: + body += rTag[0] + body += stack.spool_start() + body += rTag[1] + else: + if is_end: + t = stack.get_last() + tn = stack.get_tag_name(t) + rTn = stack.get_tag_name(rTag[0]) + if tn == rTn: + body += rTag[0] + stack.pop() + elif not is_closed: + stack.push(rTag[0]) + body += rTag[0] + else: + body += rTag[0] + + body += rTag[1] + stack.flush() + return body diff --git a/fanficdownloader/mobi.py b/fanficdownloader/mobi.py new file mode 100644 index 0000000..7a52715 --- /dev/null +++ b/fanficdownloader/mobi.py @@ -0,0 +1,386 @@ +#!/usr/bin/python +# Copyright(c) 2009 Andrew Chatham and Vijay Pandurangan + + +import StringIO +import struct +import time +import random +import logging + +logger = logging.getLogger(__name__) + +from html import HtmlProcessor + +# http://wiki.mobileread.com/wiki/MOBI +# http://membres.lycos.fr/microfirst/palm/pdb.html + +encoding = { + 'UTF-8' : 65001, + 'latin-1' : 1252, +} + +languages = {"en-us" : 0x0409, + "sv" : 0x041d, + "fi" : 0x000b, + "en" : 0x0009, + "en-gb" : 0x0809} + +def ToHex(s): + v = ['%.2x' % ord(c) for c in s] + return ' '.join(v) + +class _SubEntry: + def __init__(self, pos, html_data): + self.pos = pos + self.html = HtmlProcessor(html_data) + self.title = self.html.title + self._name = 'mobi_article_%d' % pos + if not self.title: + self.title = 'Article %d' % self.pos + + def TocLink(self): + return '<a href="#%s_MOBI_START">%.80s</a>' % (self._name, self.title) + + def Anchor(self): + return '<a name="%s_MOBI_START">' % self._name + + def Body(self): + return self.html.RenameAnchors(self._name + '_') + +class Converter: + def __init__(self, refresh_url='', title='Unknown', author='Unknown', publisher='Unknown'): + self._header = Header() + self._header.SetTitle(title) + self._header.SetAuthor(author) + self._header.SetPublisher(publisher) + self._refresh_url = refresh_url + + def ConvertString(self, s): + out = StringIO.StringIO() + self._ConvertStringToFile(s, out) + return out.getvalue() + + def ConvertStrings(self, html_strs): + out = StringIO.StringIO() + self._ConvertStringsToFile(html_strs, out) + return out.getvalue() + + def ConvertFile(self, html_file, out_file): + self._ConvertStringToFile(open(html_file,'rb').read(), + open(out_file, 'wb')) + + def ConvertFiles(self, html_files, out_file): + html_strs = [open(f,'rb').read() for f in html_files] + self._ConvertStringsToFile(html_strs, open(out_file, 'wb')) + + def MakeOneHTML(self, html_strs): + """This takes a list of HTML strings and returns a big HTML file with + all contents consolidated. It constructs a table of contents and adds + anchors within the text + """ + title_html = [] + toc_html = [] + body_html = [] + + PAGE_BREAK = '<mbp:pagebreak>' + + # pull out the title page, assumed first html_strs. + htmltitle = html_strs[0] + entrytitle = _SubEntry(1, htmltitle) + title_html.append(entrytitle.Body()) + + title_html.append(PAGE_BREAK) + toc_html.append('<a name="TOCTOP"><h3>Table of Contents</h3><br />') + + for pos, html in enumerate(html_strs[1:]): + entry = _SubEntry(pos+1, html) + toc_html.append('%s<br />' % entry.TocLink()) + + # give some space between bodies of work. + body_html.append(PAGE_BREAK) + + body_html.append(entry.Anchor()) + + body_html.append(entry.Body()) + + # TODO: this title can get way too long with RSS feeds. Not sure how to fix + # cheat slightly and use the <a href> code to set filepos in references. + header = '''<html> +<head> +<title>Bibliorize %s GMT + + + + + +''' % time.ctime(time.time()) + + footer = '' + all_html = header + '\n'.join(title_html + toc_html + body_html) + footer + #print "%s" % all_html.encode('utf8') + return all_html + + def _ConvertStringsToFile(self, html_strs, out_file): + try: + tmp = self.MakeOneHTML(html_strs) + self._ConvertStringToFile(tmp, out_file) + except Exception, e: + logger.error('Error %s', e) + #logger.debug('Details: %s' % html_strs) + + def _ConvertStringToFile(self, html_data, out): + html = HtmlProcessor(html_data) + data = html.CleanHtml() + + # collect offsets of '' tags, use to make index list. + # indexlist = [] # list of (offset,length) tuples. + # not in current use. + + # j=0 + # lastj=0 + # while True: + # j=data.find('',lastj+10) # plus a bit so we find the next. + # if j < 0: + # break + # indexlist.append((lastj,j-lastj)) + # print "index offset: %d length: %d" % (lastj,j-lastj) + # lastj=j + + records = [] +# title = html.title +# if title: +# self._header.SetTitle(title) + record_id = 1 + for start_pos in range(0, len(data), Record.MAX_SIZE): + end = min(len(data), start_pos + Record.MAX_SIZE) + record_data = data[start_pos:end] + records.append(self._header.AddRecord(record_data, record_id)) + #print "HTML Record %03d: (size:%d) [[%s ... %s]]" % ( record_id, len(record_data), record_data[:20], record_data[-20:] ) + record_id += 1 + self._header.SetImageRecordIndex(record_id) + records[0:0] = [self._header.MobiHeader()] + + header, rec_offset = self._header.PDBHeader(len(records)) + out.write(header) + for record in records: + record.WriteHeader(out, rec_offset) + #print "rec_offset: %d len(record.data): %d" % (rec_offset,len(record.data)) + rec_offset += (len(record.data)+1) # plus one for trailing null + + # Write to nuls for some reason + out.write('\0\0') + for record in records: + record.WriteData(out) + out.write('\0') + # needs a trailing null, I believe it indicates zero length 'overlap'. + # otherwise, the readers eat the last char of each html record. + # Calibre writes another 6-7 bytes of stuff after that, but we seem + # to be getting along without it. + +class Record: + MAX_SIZE = 4096 + INDEX_LEN = 8 + _unique_id_seed = 28 # should be arbitrary, but taken from MobiHeader + + # TODO(chatham): Record compression doesn't look that hard. + + def __init__(self, data, record_id): + assert len(data) <= self.MAX_SIZE + self.data = data + if record_id != 0: + self._id = record_id + else: + Record._unique_id_seed += 1 + self._id = 0 + + def __repr__(self): + return 'Record: id=%d len=%d' % (self._id, len(self.data)) + + def _SetUniqueId(self): + Record._unique_id_seed += 1 + # TODO(chatham): Wraparound crap + self._id = Record._unique_id_seed + + def WriteData(self, out): + out.write(self.data) + + def WriteHeader(self, out, rec_offset): + attributes = 64 # dirty? + header = struct.pack('>IbbH', + rec_offset, + attributes, + 0, self._id) + assert len(header) == Record.INDEX_LEN + out.write(header) + +EXTH_HEADER_FIELDS = { + 'author' : 100, + 'publisher' : 101, +} + +class Header: + EPOCH_1904 = 2082844800 + + def __init__(self): + self._length = 0 + self._record_count = 0 + self._title = '2008_2_34' + self._author = 'Unknown author' + self._publisher = 'Unknown publisher' + self._first_image_index = 0 + + def SetAuthor(self, author): + self._author = author.encode('ascii','ignore') + + def SetTitle(self, title): + # TODO(chatham): Reevaluate whether this needs to be ASCII. + # maybe just do sys.setdefaultencoding('utf-8')? Problems + # appending self._title with other things. + self._title = title.encode('ascii','ignore') + + def SetPublisher(self, publisher): + self._publisher = publisher.encode('ascii','ignore') + + def AddRecord(self, data, record_id): + self.max_record_size = max(Record.MAX_SIZE, len(data)) + self._record_count += 1 + self._length += len(data) + return Record(data, record_id) + + def _ReplaceWord(self, data, pos, word): + return data[:pos] + struct.pack('>I', word) + data[pos+4:] + + def PalmDocHeader(self): + compression = 1 # no compression + unused = 0 + encryption_type = 0 # no ecryption + records = self._record_count + 1 # the header record itself + palmdoc_header = struct.pack('>HHIHHHH', + compression, + unused, + self._length, + records, + Record.MAX_SIZE, + encryption_type, + unused) + assert len(palmdoc_header) == 16 + return palmdoc_header + + def PDBHeader(self, num_records): + HEADER_LEN = 32+2+2+9*4 + RECORD_INDEX_HEADER_LEN = 6 + RESOURCE_INDEX_LEN = 10 + + index_len = RECORD_INDEX_HEADER_LEN + num_records * Record.INDEX_LEN + rec_offset = HEADER_LEN + index_len + 2 + + short_title = self._title[0:31] + attributes = 0 + version = 0 + ctime = self.EPOCH_1904 + int(time.time()) + mtime = self.EPOCH_1904 + int(time.time()) + backup_time = self.EPOCH_1904 + int(time.time()) + modnum = 0 + appinfo_offset = 0 + sort_offset = 0 + type = 'BOOK' + creator = 'MOBI' + id_seed = 36 + header = struct.pack('>32sHHII', + short_title, attributes, version, + ctime, mtime) + header += struct.pack('>IIII', backup_time, modnum, + appinfo_offset, sort_offset) + header += struct.pack('>4s4sI', + type, creator, id_seed) + next_record = 0 # not used? + header += struct.pack('>IH', next_record, num_records) + return header, rec_offset + + def _GetExthHeader(self): + # They set author, publisher, coveroffset, thumboffset + data = {'author' : self._author, + 'publisher' : self._publisher, + } + # Turn string type names into EXTH typeids. + r = [] + for key, value in data.items(): + typeid = EXTH_HEADER_FIELDS[key] + length_encoding_len = 8 + r.append(struct.pack('>LL', typeid, len(value) + length_encoding_len,) + value) + content = ''.join(r) + + # Pad to word boundary + while len(content) % 4: + content += '\0' + TODO_mysterious = 12 + exth = 'EXTH' + struct.pack('>LL', len(content) + TODO_mysterious, len(data)) + content + return exth + + def SetImageRecordIndex(self, idx): + self._first_image_index = idx + + def MobiHeader(self): + exth_header = self._GetExthHeader(); + palmdoc_header = self.PalmDocHeader() + + fs = 0xffffffff + + # Record 0 + header_len = 0xE4 # TODO + mobi_type = 2 # BOOK + text_encoding = encoding['UTF-8'] + unique_id = random.randint(1, 1<<32) + creator_version = 4 + reserved = '%c' % 0xff * 40 + nonbook_index = fs + full_name_offset = header_len + len(palmdoc_header) + len(exth_header) # put full name after header + language = languages['en-us'] + unused = 0 + mobi_header = struct.pack('>4sIIIII40sIIIIII', + 'MOBI', + header_len, + mobi_type, + text_encoding, + unique_id, + creator_version, + reserved, + nonbook_index, + full_name_offset, + len(self._title), + language, + fs, fs) + assert len(mobi_header) == 104 - 16 + + unknown_fields = chr(0) * 32 + drm_offset = 0 + drm_count = 0 + drm_size = 0 + drm_flags = 0 + exth_flags = 0x50 + header_end = chr(0) * 64 + mobi_header += struct.pack('>IIIIIII', + creator_version, + self._first_image_index, + fs, + unused, + fs, + unused, + exth_flags) + mobi_header += '\0' * 112 # TODO: Why this much padding? + # Set some magic offsets to be 0xFFFFFFF. + for pos in (0x94, 0x98, 0xb0, 0xb8, 0xc0, 0xc8, 0xd0, 0xd8, 0xdc): + mobi_header = self._ReplaceWord(mobi_header, pos, fs) + + # 16 bytes? + padding = '\0' * 48 * 4 # why? + total_header = palmdoc_header + mobi_header + exth_header + self._title + padding + + return self.AddRecord(total_header, 0) + +if __name__ == '__main__': + import sys + m = Converter(title='Testing Mobi', author='Mobi Author', publisher='mobi converter') + m.ConvertFiles(sys.argv[1:], 'test.mobi') + #m.ConvertFile(sys.argv[1], 'test.mobi') diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py new file mode 100644 index 0000000..d3bad27 --- /dev/null +++ b/fanficdownloader/story.py @@ -0,0 +1,873 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os, re +import urlparse +import string +from math import floor +from functools import partial +import logging +logger = logging.getLogger(__name__) +import urlparse as up + +import exceptions +from htmlcleanup import conditionalRemoveEntities, removeAllEntities +from configurable import Configurable + +SPACE_REPLACE=u'\s' +SPLIT_META=u'\,' + +# Create convert_image method depending on which graphics lib we can +# load. Preferred: calibre, PIL, none + +imagetypes = { + 'jpg':'image/jpeg', + 'jpeg':'image/jpeg', + 'png':'image/png', + 'gif':'image/gif', + 'svg':'image/svg+xml', + } + +try: + from calibre.utils.magick import Image + convtype = {'jpg':'JPG', 'png':'PNG'} + + def convert_image(url,data,sizes,grayscale, + removetrans,imgtype="jpg",background='#ffffff'): + export = False + img = Image() + img.load(data) + + owidth, oheight = img.size + nwidth, nheight = sizes + scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight) + if scaled: + img.size = (nwidth, nheight) + export = True + + if normalize_format_name(img.format) != imgtype: + export = True + + if removetrans and img.has_transparent_pixels(): + canvas = Image() + canvas.create_canvas(int(img.size[0]), int(img.size[1]), str(background)) + canvas.compose(img) + img = canvas + export = True + + if grayscale and img.type != "GrayscaleType": + img.type = "GrayscaleType" + export = True + + if export: + return (img.export(convtype[imgtype]),imgtype,imagetypes[imgtype]) + else: + logger.debug("image used unchanged") + return (data,imgtype,imagetypes[imgtype]) + +except: + + # No calibre routines, try for PIL for CLI. + try: + import Image + from StringIO import StringIO + convtype = {'jpg':'JPEG', 'png':'PNG'} + def convert_image(url,data,sizes,grayscale, + removetrans,imgtype="jpg",background='#ffffff'): + export = False + img = Image.open(StringIO(data)) + + owidth, oheight = img.size + nwidth, nheight = sizes + scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight) + if scaled: + img = img.resize((nwidth, nheight),Image.ANTIALIAS) + export = True + + if normalize_format_name(img.format) != imgtype: + if img.mode == "P": + # convert pallete gifs to RGB so jpg save doesn't fail. + img = img.convert("RGB") + export = True + + if removetrans and img.mode == "RGBA": + background = Image.new('RGBA', img.size, background) + # Paste the image on top of the background + background.paste(img, img) + img = background.convert('RGB') + export = True + + if grayscale and img.mode != "L": + img = img.convert("L") + export = True + + if export: + outsio = StringIO() + img.save(outsio,convtype[imgtype]) + return (outsio.getvalue(),imgtype,imagetypes[imgtype]) + else: + logger.debug("image used unchanged") + return (data,imgtype,imagetypes[imgtype]) + + except: + # No calibre or PIL, simple pass through with mimetype. + def convert_image(url,data,sizes,grayscale, + removetrans,imgtype="jpg",background='#ffffff'): + return no_convert_image(url,data) + +## also used for explicit no image processing. +def no_convert_image(url,data): + parsedUrl = up.urlparse(url) + + ext=parsedUrl.path[parsedUrl.path.rfind('.')+1:].lower() + + if ext not in imagetypes: + logger.debug("no_convert_image url:%s - no known extension"%url) + # doesn't have extension? use jpg. + ext='jpg' + + return (data,ext,imagetypes[ext]) + +def normalize_format_name(fmt): + if fmt: + fmt = fmt.lower() + if fmt == 'jpeg': + fmt = 'jpg' + return fmt + +def fit_image(width, height, pwidth, pheight): + ''' + Fit image in box of width pwidth and height pheight. + @param width: Width of image + @param height: Height of image + @param pwidth: Width of box + @param pheight: Height of box + @return: scaled, new_width, new_height. scaled is True iff new_width and/or new_height is different from width or height. + ''' + scaled = height > pheight or width > pwidth + if height > pheight: + corrf = pheight/float(height) + width, height = floor(corrf*width), pheight + if width > pwidth: + corrf = pwidth/float(width) + width, height = pwidth, floor(corrf*height) + if height > pheight: + corrf = pheight/float(height) + width, height = floor(corrf*width), pheight + + return scaled, int(width), int(height) + +try: + # doesn't really matter what, just checking for appengine. + from google.appengine.api import apiproxy_stub_map + + is_appengine = True +except: + is_appengine = False + + +# The list comes from ffnet, the only multi-language site we support +# at the time of writing. Values are taken largely from pycountry, +# but with some corrections and guesses. +langs = { + "English":"en", + "Spanish":"es", + "French":"fr", + "German":"de", + "Chinese":"zh", + "Japanese":"ja", + "Dutch":"nl", + "Portuguese":"pt", + "Russian":"ru", + "Italian":"it", + "Bulgarian":"bg", + "Polish":"pl", + "Hungarian":"hu", + "Hebrew":"he", + "Arabic":"ar", + "Swedish":"sv", + "Norwegian":"no", + "Danish":"da", + "Finnish":"fi", + "Filipino":"fil", + "Esperanto":"eo", + "Hindi":"hi", + "Punjabi":"pa", + "Farsi":"fa", + "Greek":"el", + "Romanian":"ro", + "Albanian":"sq", + "Serbian":"sr", + "Turkish":"tr", + "Czech":"cs", + "Indonesian":"id", + "Croatian":"hr", + "Catalan":"ca", + "Latin":"la", + "Korean":"ko", + "Vietnamese":"vi", + "Thai":"th", + "Devanagari":"hi", + } + +def re_compile(regex,line): + try: + return re.compile(regex) + except Exception, e: + raise exceptions.RegularExpresssionFailed(e,regex,line) + +class InExMatch: + keys = [] + regex = None + match = None + negate = False + + def __init__(self,line): + if "=~" in line: + (self.keys,self.match) = line.split("=~") + self.match = self.match.replace(SPACE_REPLACE,' ') + self.regex = re_compile(self.match,line) + elif "!~" in line: + (self.keys,self.match) = line.split("!~") + self.match = self.match.replace(SPACE_REPLACE,' ') + self.regex = re_compile(self.match,line) + self.negate = True + elif "==" in line: + (self.keys,self.match) = line.split("==") + self.match = self.match.replace(SPACE_REPLACE,' ') + elif "!=" in line: + (self.keys,self.match) = line.split("!=") + self.match = self.match.replace(SPACE_REPLACE,' ') + self.negate = True + self.keys = map( lambda x: x.strip(), self.keys.split(",") ) + + # For conditional, only one key + def is_key(self,key): + return key == self.keys[0] + + # For conditional, only one key + def key(self): + return self.keys[0] + + def in_keys(self,key): + return key in self.keys + + def is_match(self,value): + retval = False + if self.regex: + if self.regex.search(value): + retval = True + #print(">>>>>>>>>>>>>%s=~%s r: %s,%s=%s"%(self.match,value,self.negate,retval,self.negate != retval)) + else: + retval = self.match == value + #print(">>>>>>>>>>>>>%s==%s r: %s,%s=%s"%(self.match,value,self.negate,retval, self.negate != retval)) + + return self.negate != retval + + def __str__(self): + if self.negate: + f='!' + else: + f='=' + if self.regex: + s='~' + else: + s='=' + return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match) + +class Story(Configurable): + + def __init__(self, configuration): + Configurable.__init__(self, configuration) + try: + ## calibre plugin will set externally to match PI version. + self.metadata = {'version':os.environ['CURRENT_VERSION_ID']} + except: + self.metadata = {'version':'4.4'} + self.replacements = [] + self.in_ex_cludes = {} + self.chapters = [] # chapters will be tuples of (title,html) + self.imgurls = [] + self.imgtuples = [] + + self.cover=None # *href* of new cover image--need to create html. + self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata) + self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update. + self.logfile=None # cheesy way to carry log file forward across update. + + ## Look for config parameter, split and add each to metadata field. + for (config,metadata) in [("extracategories","category"), + ("extragenres","genre"), + ("extracharacters","characters"), + ("extraships","ships"), + ("extrawarnings","warnings")]: + for val in self.getConfigList(config): + self.addToList(metadata,val) + + self.setReplace(self.getConfig('replace_metadata')) + + in_ex_clude_list = ['include_metadata_pre','exclude_metadata_pre', + 'include_metadata_post','exclude_metadata_post'] + for ie in in_ex_clude_list: + ies = self.getConfig(ie) + # print("%s %s"%(ie,ies)) + if ies: + iel = [] + self.in_ex_cludes[ie] = self.set_in_ex_clude(ies) + + def join_list(self, key, vallist): + return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist)) + + def setMetadata(self, key, value, condremoveentities=True): + + # keep as list type, but set as only value. + if self.isList(key): + self.addToList(key,value,condremoveentities=condremoveentities,clear=True) + else: + ## still keeps < < and & + if condremoveentities: + self.metadata[key]=conditionalRemoveEntities(value) + else: + self.metadata[key]=value + + if key == "language": + try: + # getMetadata not just self.metadata[] to do replace_metadata. + self.setMetadata('langcode',langs[self.getMetadata(key)]) + except: + self.setMetadata('langcode','en') + + if key == 'dateUpdated' and value: + # Last Update tags for Bill. + self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m")) + self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d")) + + + ## metakey[,metakey]=~pattern + ## metakey[,metakey]==string + ## *for* part lines. Effect only when trailing conditional key=~regexp matches + ## metakey[,metakey]=~pattern[&&metakey=~regexp] + ## metakey[,metakey]==string[&&metakey=~regexp] + ## metakey[,metakey]=~pattern[&&metakey==string] + ## metakey[,metakey]==string[&&metakey==string] + def set_in_ex_clude(self,setting): + dest = [] + # print("set_in_ex_clude:"+setting) + for line in setting.splitlines(): + if line: + (match,condmatch)=(None,None) + if "&&" in line: + (line,conditional) = line.split("&&") + condmatch = InExMatch(conditional) + match = InExMatch(line) + dest.append([match,condmatch]) + return dest + + def do_in_ex_clude(self,which,value,key): + if value and which in self.in_ex_cludes: + include = 'include' in which + keyfound = False + found = False + for (match,condmatch) in self.in_ex_cludes[which]: + keyfndnow = False + if match.in_keys(key): + # key in keys and either no conditional, or conditional matched + if condmatch == None or condmatch.is_key(key): + keyfndnow = True + else: + condval = self.getMetadata(condmatch.key()) + keyfndnow = condmatch.is_match(condval) + keyfound |= keyfndnow + # print("match:%s %s\ncondmatch:%s %s\n\tkeyfound:%s\n\tfound:%s"%( + # match,value,condmatch,condval,keyfound,found)) + if keyfndnow: + found = isinstance(value,basestring) and match.is_match(value) + if found: + # print("match:%s %s\n\tkeyfndnow:%s\n\tfound:%s"%( + # match,value,keyfndnow,found)) + if not include: + value = None + break + if include and keyfound and not found: + value = None + return value + + + ## Two or three part lines. Two part effect everything. + ## Three part effect only those key(s) lists. + ## pattern=>replacement + ## metakey,metakey=>pattern=>replacement + ## *Five* part lines. Effect only when trailing conditional key=>regexp matches + ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp] + def setReplace(self,replace): + for line in replace.splitlines(): + # print("replacement line:%s"%line) + (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None) + if "&&" in line: + (line,conditional) = line.split("&&") + (condkey,condregexp) = conditional.split("=>") + if "=>" in line: + parts = line.split("=>") + if len(parts) > 2: + metakeys = map( lambda x: x.strip(), parts[0].split(",") ) + (regexp,replacement)=parts[1:] + else: + (regexp,replacement)=parts + + if regexp: + regexp = re_compile(regexp,line) + if condregexp: + condregexp = re_compile(condregexp,line) + # A way to explicitly include spaces in the + # replacement string. The .ini parser eats any + # trailing spaces. + replacement=replacement.replace(SPACE_REPLACE,' ') + self.replacements.append([metakeys,regexp,replacement,condkey,condregexp]) + + def doReplacements(self,value,key,return_list=False,seen_list=[]): + value = self.do_in_ex_clude('include_metadata_pre',value,key) + value = self.do_in_ex_clude('exclude_metadata_pre',value,key) + + retlist = [value] + for replaceline in self.replacements: + if replaceline in seen_list: # recursion on pattern, bail + # print("bailing on %s"%replaceline) + continue + #print("replacement tuple:%s"%replaceline) + (metakeys,regexp,replacement,condkey,condregexp) = replaceline + if (metakeys == None or key in metakeys) \ + and isinstance(value,basestring) \ + and regexp.search(value): + doreplace=True + if condkey and condkey != key: # prevent infinite recursion. + condval = self.getMetadata(condkey) + doreplace = condval != None and condregexp.search(condval) + + if doreplace: + # split into more than one list entry if + # SPLIT_META present in replacement string. Split + # first, then regex sub, then recurse call replace + # on each. Break out of loop, each split element + # handled individually by recursion call. + if SPLIT_META in replacement: + retlist = [] + for splitrepl in replacement.split(SPLIT_META): + retlist.extend(self.doReplacements(regexp.sub(splitrepl,value), + key, + return_list=True, + seen_list=seen_list+[replaceline])) + break + else: + # print("replacement,value:%s,%s->%s"%(replacement,value,regexp.sub(replacement,value))) + value = regexp.sub(replacement,value) + retlist = [value] + + for val in retlist: + retlist = map(partial(self.do_in_ex_clude,'include_metadata_post',key=key),retlist) + retlist = map(partial(self.do_in_ex_clude,'exclude_metadata_post',key=key),retlist) + # value = self.do_in_ex_clude('include_metadata_post',value,key) + # value = self.do_in_ex_clude('exclude_metadata_post',value,key) + + if return_list: + return retlist + else: + return self.join_list(key,retlist) + + def getMetadataRaw(self,key): + if self.isValidMetaEntry(key) and self.metadata.has_key(key): + return self.metadata[key] + + def getMetadata(self, key, + removeallentities=False, + doreplacements=True): + value = None + if not self.isValidMetaEntry(key): + return value + + if self.isList(key): + # join_string = self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ') + # value = join_string.join(self.getList(key, removeallentities, doreplacements=True)) + value = self.join_list(key,self.getList(key, removeallentities, doreplacements=True)) + if doreplacements: + value = self.doReplacements(value,key+"_LIST") + return value + elif self.metadata.has_key(key): + value = self.metadata[key] + if value: + if key == "numWords": + value = commaGroups(value) + if key == "numChapters": + value = commaGroups("%d"%value) + if key in ("dateCreated"): + value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d %H:%M:%S")) + if key in ("datePublished","dateUpdated"): + value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d")) + + if doreplacements: + value=self.doReplacements(value,key) + if removeallentities and value != None: + return removeAllEntities(value) + else: + return value + else: #if self.getConfig("default_value_"+key): + return self.getConfig("default_value_"+key) + + def getAllMetadata(self, + removeallentities=False, + doreplacements=True, + keeplists=False): + ''' + All single value *and* list value metadata as strings (unless + keeplists=True, then keep lists). + ''' + allmetadata = {} + + # special handling for authors/authorUrls + linkhtml="%s" + if self.isList('author'): # more than one author, assume multiple authorUrl too. + htmllist=[] + for i, v in enumerate(self.getList('author')): + aurl = self.getList('authorUrl')[i] + auth = v + # make sure doreplacements & removeallentities are honored. + if doreplacements: + aurl=self.doReplacements(aurl,'authorUrl') + auth=self.doReplacements(auth,'author') + if removeallentities: + aurl=removeAllEntities(aurl) + auth=removeAllEntities(auth) + + htmllist.append(linkhtml%('author',aurl,auth)) + # join_string = self.getConfig("join_string_authorHTML",u", ").replace(SPACE_REPLACE,' ') + self.setMetadata('authorHTML',self.join_list("join_string_authorHTML",htmllist)) + else: + self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements), + self.getMetadata('author', removeallentities, doreplacements))) + + self.extendList("extratags",self.getConfigList("extratags")) + + if self.getMetadataRaw('seriesUrl'): + self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements), + self.getMetadata('series', removeallentities, doreplacements))) + elif self.getMetadataRaw('series'): + self.setMetadata('seriesHTML',self.getMetadataRaw('series')) + + # logger.debug("make_linkhtml_entries:%s"%self.getConfig('make_linkhtml_entries')) + for k in self.getConfigList('make_linkhtml_entries'): + # Assuming list, because it has to be site specific and + # they are all lists. Bail if kUrl list not the same + # length. + # logger.debug("\nk:%s\nlist:%s\nlistURL:%s"%(k,self.getList(k),self.getList(k+'Url'))) + if len(self.getList(k+'Url')) != len(self.getList(k)): + continue + htmllist=[] + for i, v in enumerate(self.getList(k)): + url = self.getList(k+'Url')[i] + # make sure doreplacements & removeallentities are honored. + if doreplacements: + url=self.doReplacements(url,k+'Url') + v=self.doReplacements(v,k) + if removeallentities: + url=removeAllEntities(url) + v=removeAllEntities(v) + + htmllist.append(linkhtml%(k,url,v)) + # join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace(SPACE_REPLACE,' ') + self.setMetadata(k+'HTML',self.join_list("join_string_"+k+"HTML",htmllist)) + + for k in self.getValidMetaList(): + if self.isList(k) and keeplists: + allmetadata[k] = self.getList(k, removeallentities, doreplacements) + else: + allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements) + + return allmetadata + + # just for less clutter in adapters. + def extendList(self,listname,l): + for v in l: + self.addToList(listname,v.strip()) + + def addToList(self,listname,value,condremoveentities=True,clear=False): + if value==None: + return + if condremoveentities: + value = conditionalRemoveEntities(value) + if clear or not self.isList(listname) or not listname in self.metadata: + # Calling addToList to a non-list meta will overwrite it. + self.metadata[listname]=[] + # prevent duplicates. + if not value in self.metadata[listname]: + self.metadata[listname].append(value) + + if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1: + self.addToList('genre',self.getConfig('add_genre_when_multi_category')) + + def isList(self,listname): + 'Everything set with an include_in_* is considered a list.' + return self.isListType(listname) or \ + ( self.isValidMetaEntry(listname) and self.metadata.has_key(listname) \ + and isinstance(self.metadata[listname],list) ) + + def getList(self,listname, + removeallentities=False, + doreplacements=True, + includelist=[]): + #print("getList(%s,%s)"%(listname,includelist)) + retlist = [] + + if not self.isValidMetaEntry(listname): + return retlist + + # includelist prevents infinite recursion of include_in_'s + if self.hasConfig("include_in_"+listname) and listname not in includelist: + for k in self.getConfigList("include_in_"+listname): + retlist.extend(self.getList(k,removeallentities=False, + doreplacements=doreplacements,includelist=includelist+[listname])) + else: + + if not self.isList(listname): + retlist = [self.getMetadata(listname,removeallentities=False, + doreplacements=doreplacements)] + else: + retlist = self.getMetadataRaw(listname) + + if retlist: + if doreplacements: + newretlist = [] + for val in retlist: + newretlist.extend(self.doReplacements(val,listname,return_list=True)) + retlist = newretlist + + if removeallentities: + retlist = map(removeAllEntities,retlist) + + retlist = filter( lambda x : x!=None and x!='' ,retlist) + + # reorder ships so b/a and c/b/a become a/b and a/b/c. Only on '/', + # use replace_metadata to change separator first if needed. + # ships=>[ ]*(/|&|&)[ ]*=>/ + if listname == 'ships' and self.getConfig('sort_ships') and retlist: + retlist = [ '/'.join(sorted(x.split('/'))) for x in retlist ] + + if retlist: + if listname in ('author','authorUrl','authorId') or self.getConfig('keep_in_order_'+listname): + # need to retain order for author & authorUrl so the + # two match up. + return retlist + else: + # remove dups and sort. + return sorted(list(set(retlist))) + else: + return [] + + def getSubjectTags(self, removeallentities=False): + # set to avoid duplicates subject tags. + subjectset = set() + + tags_list = self.getConfigList("include_subject_tags") + self.getConfigList("extra_subject_tags") + + # metadata all go into dc:subject tags, but only if they are configured. + for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems(): + if name in tags_list: + if isinstance(value,list): + for tag in value: + subjectset.add(tag) + else: + subjectset.add(value) + + if None in subjectset: + subjectset.remove(None) + if '' in subjectset: + subjectset.remove('') + + return list(subjectset | set(self.getConfigList("extratags"))) + + def addChapter(self, url, title, html): + if self.getConfig('strip_chapter_numbers') and \ + self.getConfig('chapter_title_strip_pattern'): + title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title) + self.chapters.append( (url,title,html) ) + + def getChapters(self,fortoc=False): + "Chapters will be tuples of (title,html)" + retval = [] + ## only add numbers if more than one chapter. + if len(self.chapters) > 1 and \ + (self.getConfig('add_chapter_numbers') == "true" \ + or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \ + and self.getConfig('chapter_title_add_pattern'): + for index, (url,title,html) in enumerate(self.chapters): + retval.append( (url, + string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}), + html) ) + else: + retval = self.chapters + + return retval + + def formatFileName(self,template,allowunsafefilename=True): + values = origvalues = self.getAllMetadata() + # fall back default: + if not template: + template="${title}-${siteabbrev}_${storyId}${formatext}" + + if not allowunsafefilename: + values={} + pattern = re_compile(self.getConfig("output_filename_safepattern",r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+"),"output_filename_safepattern") + for k in origvalues.keys(): + values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k))) + + return string.Template(template).substitute(values).encode('utf8') + + # pass fetch in from adapter in case we need the cookies collected + # as well as it's a base_story class method. + def addImgUrl(self,parenturl,url,fetch,cover=False,coverexclusion=None): + + # otherwise it saves the image in the epub even though it + # isn't used anywhere. + if cover and self.getConfig('never_make_cover'): + return (None,None) + + url = url.strip() # ran across an image with a space in the + # src. Browser handled it, so we'd better, too. + + # appengine (web version) isn't allowed to do images--just + # gets too big too fast and breaks things. + if is_appengine: + return (None,None) + + if url.startswith("http") or url.startswith("file") or parenturl == None: + imgurl = url + else: + parsedUrl = urlparse.urlparse(parenturl) + if url.startswith("//") : + imgurl = urlparse.urlunparse( + (parsedUrl.scheme, + '', + url, + '','','')) + elif url.startswith("/") : + imgurl = urlparse.urlunparse( + (parsedUrl.scheme, + parsedUrl.netloc, + url, + '','','')) + else: + toppath="" + if parsedUrl.path.endswith("/"): + toppath = parsedUrl.path + else: + toppath = parsedUrl.path[:parsedUrl.path.rindex('/')] + imgurl = urlparse.urlunparse( + (parsedUrl.scheme, + parsedUrl.netloc, + toppath + '/' + url, + '','','')) + #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl)) + + # apply coverexclusion to explicit covers, too. Primarily for ffnet imageu. + if cover and coverexclusion and re.search(coverexclusion,imgurl): + return (None,None) + + prefix='ffdl' + if imgurl not in self.imgurls: + parsedUrl = urlparse.urlparse(imgurl) + + try: + if self.getConfig('no_image_processing'): + (data,ext,mime) = no_convert_image(imgurl, + fetch(imgurl)) + else: + try: + sizes = [ int(x) for x in self.getConfigList('image_max_size') ] + except Exception, e: + raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(self.getConfigList('image_max_size'),e)) + grayscale = self.getConfig('grayscale_images') + imgtype = self.getConfig('convert_images_to') + if not imgtype: + imgtype = "jpg" + removetrans = self.getConfig('remove_transparency') + removetrans = removetrans or grayscale or imgtype=="jpg" + (data,ext,mime) = convert_image(imgurl, + fetch(imgurl), + sizes, + grayscale, + removetrans, + imgtype, + background="#"+self.getConfig('background_color')) + except Exception, e: + logger.info("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e)) + return ("failedtoload","failedtoload") + + # explicit cover, make the first image. + if cover: + if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']: + # remove existing cover, if there is one. + del self.imgurls[0] + del self.imgtuples[0] + self.imgurls.insert(0,imgurl) + newsrc = "images/cover.%s"%ext + self.cover=newsrc + self.imgtuples.insert(0,{'newsrc':newsrc,'mime':mime,'data':data}) + else: + self.imgurls.append(imgurl) + # First image, copy not link because calibre will replace with it's cover. + # Only if: No cover already AND + # make_firstimage_cover AND + # NOT never_make_cover AND + # either no coverexclusion OR coverexclusion doesn't match + if self.cover == None and \ + self.getConfig('make_firstimage_cover') and \ + not self.getConfig('never_make_cover') and \ + not (coverexclusion and re.search(coverexclusion,imgurl)): + newsrc = "images/cover.%s"%ext + self.cover=newsrc + self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data}) + self.imgurls.append(imgurl) + + newsrc = "images/%s-%s.%s"%( + prefix, + self.imgurls.index(imgurl), + ext) + self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data}) + + #logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data))) + else: + newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc'] + + #print("===============\n%s\nimg url:%s\n============"%(newsrc,self.imgurls[-1])) + + return (newsrc, imgurl) + + def getImgUrls(self): + retlist = [] + for i, url in enumerate(self.imgurls): + #parsedUrl = urlparse.urlparse(url) + retlist.append(self.imgtuples[i]) + return retlist + + def __str__(self): + return "Metadata: " +str(self.metadata) + +def commaGroups(s): + groups = [] + while s and s[-1].isdigit(): + groups.append(s[-3:]) + s = s[:-3] + return s + ','.join(reversed(groups)) + diff --git a/fanficdownloader/translit.py b/fanficdownloader/translit.py new file mode 100644 index 0000000..bf205a6 --- /dev/null +++ b/fanficdownloader/translit.py @@ -0,0 +1,57 @@ +#-*-coding:utf-8-*- +# Code taken from http://python.su/forum/viewtopic.php?pid=66946 +import unicodedata +def is_syllable(letter): + syllables = ("A", "E", "I", "O", "U", "a", "e", "i", "o", "u") + if letter in syllables: + return True + return False +def is_consonant(letter): + return not is_syllable(letter) +def romanize(letter): + try: + str(letter) + except UnicodeEncodeError: + pass + else: + return str(letter) + unid = unicodedata.name(letter) + exceptions = {"NUMERO SIGN": "No", "LEFT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "DASH": "-"} + for name_contains in exceptions: + if unid.find(name_contains)!=-1: + return exceptions[name_contains] + assert(unid.startswith("CYRILLIC"))# Not ready to romanize anything but cyrillics + transformation_pairs = {"CYRILLIC CAPITAL LETTER ": str.capitalize, "CYRILLIC SMALL LETTER ": str.lower} + func = str.lower + for name_contains in transformation_pairs: + if unid.find(name_contains)!=-1: + func = transformation_pairs[name_contains] + unid = unid.replace(name_contains, "") + cyrillic_exceptions = {"YERU": "y", "SHORT I": "y", "HARD SIGN": "\'", "SOFT SIGN": "\'", "BYELORUSSIAN-UKRAINIAN I": "i", "GHE WITH UPTURN": "g", "UKRAINIAN IE": "ie", "YU": "yu", "YA": "ya"} + for name_contains in cyrillic_exceptions: + if unid.find(name_contains)!=-1: + return cyrillic_exceptions[name_contains] + if all(map(is_syllable, unid)): + return func(unid) + else: + return func(filter(is_consonant, unid)) +def translit(text): + output = "" + for letter in text: + output += romanize(letter) + return output +#def main(): + #text = u"русск.: Любя, съешь щипцы, — вздохнёт мэр, — кайф жгуч." + #print translit(text) + #text = u"укр.: Гей, хлопці, не вспію - на ґанку ваша файна їжа знищується бурундучком." + #print translit(text) + #text = u"болг.: Ах, чудна българска земьо, полюшквай цъфтящи жита." + #print translit(text) + #text = u"серб.: Неуредне ноћне даме досађивале су Џеку К." + #print translit(text) + #russk.: Lyubya, s'iesh' shchiptsy, - vzdohniot mer, - kayf zhghuch. + #ukr.: Ghiey, hloptsi, nie vspiyu - na ganku vasha fayna yzha znishchuiet'sya burunduchkom. + #bolgh.: Ah, chudna b'lgharska ziem'o, polyushkvay ts'ftyashchi zhita. + #sierb.: Nieuriednie notshnie damie dosadjivalie su Dzhieku K. +if __name__=="__main__": + main() \ No newline at end of file diff --git a/fanficdownloader/writers/__init__.py b/fanficdownloader/writers/__init__.py new file mode 100644 index 0000000..7d9faf6 --- /dev/null +++ b/fanficdownloader/writers/__init__.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## This could (should?) use a dynamic loader like adapters, but for +## now, it's static, since there's so few of them. + +from ..exceptions import FailedToDownload + +from writer_html import HTMLWriter +from writer_txt import TextWriter +from writer_epub import EpubWriter +from writer_mobi import MobiWriter + +def getWriter(type,config,story): + if type == "html": + return HTMLWriter(config,story) + if type == "txt": + return TextWriter(config,story) + if type == "epub": + return EpubWriter(config,story) + if type == "mobi": + return MobiWriter(config,story) + + raise FailedToDownload("(%s) is not a supported download format."%type) diff --git a/fanficdownloader/writers/base_writer.py b/fanficdownloader/writers/base_writer.py new file mode 100644 index 0000000..c26d409 --- /dev/null +++ b/fanficdownloader/writers/base_writer.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import os.path +import datetime +import string +import StringIO +import zipfile +from zipfile import ZipFile, ZIP_DEFLATED +import logging + +from ..configurable import Configurable +from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML + +logger = logging.getLogger(__name__) + +class BaseStoryWriter(Configurable): + + @staticmethod + def getFormatName(): + return 'base' + + @staticmethod + def getFormatExt(): + return '.bse' + + def __init__(self, configuration, adapter): + Configurable.__init__(self, configuration) + + self.adapter = adapter + self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially. + + # fall back labels. + self.titleLabels = { + 'category':'Category', + 'genre':'Genre', + 'language':'Language', + 'status':'Status', + 'series':'Series', + 'characters':'Characters', + 'ships':'Relationships', + 'datePublished':'Published', + 'dateUpdated':'Updated', + 'dateCreated':'Packaged', + 'rating':'Rating', + 'warnings':'Warnings', + 'numChapters':'Chapters', + 'numWords':'Words', + 'site':'Site', + 'storyId':'Story ID', + 'authorId':'Author ID', + 'extratags':'Extra Tags', + 'title':'Title', + 'storyUrl':'Story URL', + 'description':'Summary', + 'author':'Author', + 'authorUrl':'Author URL', + 'formatname':'File Format', + 'formatext':'File Extension', + 'siteabbrev':'Site Abbrev', + 'version':'FFDL Version' + } + self.story.setMetadata('formatname',self.getFormatName()) + self.story.setMetadata('formatext',self.getFormatExt()) + + def getMetadata(self,key, removeallentities=False): + return stripHTML(self.story.getMetadata(key, removeallentities)) + + def getOutputFileName(self): + if self.getConfig('zip_output'): + return self.getZipFileName() + else: + return self.getBaseFileName() + + def getBaseFileName(self): + return self.story.formatFileName(self.getConfig('output_filename'),self.getConfig('allow_unsafe_filename')) + + def getZipFileName(self): + return self.story.formatFileName(self.getConfig('zip_filename'),self.getConfig('allow_unsafe_filename')) + + def _write(self, out, text): + out.write(text.encode('utf8')) + + def writeTitlePage(self, out, START, ENTRY, END, WIDE_ENTRY=None, NO_TITLE_ENTRY=None): + """ + Write the title page, but only include entries that there's + metadata for. START, ENTRY and END are expected to already by + string.Template(). START and END are expected to use the same + names as Story.metadata, but ENTRY should use label and value. + """ + if self.getConfig("include_titlepage"): + + if self.hasConfig("titlepage_start"): + START = string.Template(self.getConfig("titlepage_start")) + + if self.hasConfig("titlepage_entry"): + ENTRY = string.Template(self.getConfig("titlepage_entry")) + + if self.hasConfig("titlepage_end"): + END = string.Template(self.getConfig("titlepage_end")) + + if self.hasConfig("titlepage_wide_entry"): + WIDE_ENTRY = string.Template(self.getConfig("titlepage_wide_entry")) + + if self.hasConfig("titlepage_no_title_entry"): + NO_TITLE_ENTRY = string.Template(self.getConfig("titlepage_no_title_entry")) + + self._write(out,START.substitute(self.story.getAllMetadata())) + + if WIDE_ENTRY==None: + WIDE_ENTRY=ENTRY + + titleEntriesList = self.getConfigList("titlepage_entries") + self.getConfigList("extra_titlepage_entries") + wideTitleEntriesList = self.getConfigList("wide_titlepage_entries") + + for entry in titleEntriesList: + if self.isValidMetaEntry(entry): + if self.story.getMetadata(entry): + if entry in wideTitleEntriesList: + TEMPLATE=WIDE_ENTRY + else: + TEMPLATE=ENTRY + + if self.hasConfig(entry+"_label"): + label=self.getConfig(entry+"_label") + elif entry in self.titleLabels: + logger.debug("Using fallback label for %s_label"%entry) + label=self.titleLabels[entry] + else: + label="%s"%entry.title() + logger.debug("No known label for %s, fallback to '%s'"%(entry,label)) + + # If the label for the title entry is empty, use the + # 'no title' option if there is one. + if label == "" and NO_TITLE_ENTRY: + TEMPLATE= NO_TITLE_ENTRY + + self._write(out,TEMPLATE.substitute({'label':label, + 'id':entry, + 'value':self.story.getMetadata(entry)})) + else: + self._write(out, entry) + + self._write(out,END.substitute(self.story.getAllMetadata())) + + def writeTOCPage(self, out, START, ENTRY, END): + """ + Write the Table of Contents page. START, ENTRY and END are expected to already by + string.Template(). START and END are expected to use the same + names as Story.metadata, but ENTRY should use index and chapter. + """ + # Only do TOC if there's more than one chapter and it's configured. + if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly : + if self.hasConfig("tocpage_start"): + START = string.Template(self.getConfig("tocpage_start")) + + if self.hasConfig("tocpage_entry"): + ENTRY = string.Template(self.getConfig("tocpage_entry")) + + if self.hasConfig("tocpage_end"): + END = string.Template(self.getConfig("tocpage_end")) + + self._write(out,START.substitute(self.story.getAllMetadata())) + + for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)): + if html: + self._write(out,ENTRY.substitute({'chapter':title, + 'number':index+1, + 'index':"%04d"%(index+1), + 'url':url})) + + self._write(out,END.substitute(self.story.getAllMetadata())) + + # if no outstream is given, write to file. + def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False): + + self.metaonly = metaonly + if outfilename == None: + outfilename=self.getOutputFileName() + + self.outfilename = outfilename + + # minor cheat, tucking css into metadata. + if self.getConfig("output_css"): + self.story.setMetadata("output_css", + self.getConfig("output_css"), + condremoveentities=False) + else: + self.story.setMetadata("output_css",'') + + if not outstream: + close=True + logger.info("Save directly to file: %s" % outfilename) + if self.getConfig('make_directories'): + path="" + outputdirs = os.path.dirname(outfilename).split('/') + for dir in outputdirs: + path+=dir+"/" + if not os.path.exists(path): + os.mkdir(path) ## os.makedirs() doesn't work in 2.5.2? + + ## Check for output file date vs updated date here + if not (self.getConfig('always_overwrite') or forceOverwrite): + if os.path.exists(outfilename): + ## date() truncs off time, which files have, but sites don't report. + lastupdated=self.story.getMetadataRaw('dateUpdated').date() + fileupdated=datetime.datetime.fromtimestamp(os.stat(outfilename)[8]).date() + if fileupdated > lastupdated: + logger.warn("File(%s) Updated(%s) more recently than Story(%s) - Skipping" % (outfilename,fileupdated,lastupdated)) + return + if not metaonly: + self.story = self.adapter.getStory() # get full story + # now, just + # before writing. + # Fetch before + # opening file. + outstream = open(outfilename,"wb") + else: + close=False + logger.debug("Save to stream") + + if not metaonly: + self.story = self.adapter.getStory() # get full story now, + # just before + # writing. Okay if + # double called with + # above, it will only + # fetch once. + if self.getConfig('zip_output'): + out = StringIO.StringIO() + self.zipout = ZipFile(outstream, 'w', compression=ZIP_DEFLATED) + self.writeStoryImpl(out) + self.zipout.writestr(self.getBaseFileName(),out.getvalue()) + # declares all the files created by Windows. otherwise, when + # it runs in appengine, windows unzips the files as 000 perms. + for zf in self.zipout.filelist: + zf.create_system = 0 + self.zipout.close() + out.close() + else: + self.writeStoryImpl(outstream) + + if close: + outstream.close() + + def writeFile(self, filename, data): + logger.debug("writeFile:%s"%filename) + + if self.getConfig('zip_output'): + outputdirs = os.path.dirname(self.getBaseFileName()) + if outputdirs: + filename=outputdirs+'/'+filename + self.zipout.writestr(filename,data) + else: + outputdirs = os.path.dirname(self.outfilename) + if outputdirs: + filename=outputdirs+'/'+filename + + dir = os.path.dirname(filename) + if not os.path.exists(dir): + os.mkdir(dir) ## os.makedirs() doesn't work in 2.5.2? + + outstream = open(filename,"wb") + outstream.write(data) + outstream.close() + + def writeStoryImpl(self, out): + "Must be overriden by sub classes." + pass + diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py new file mode 100644 index 0000000..3b1b19d --- /dev/null +++ b/fanficdownloader/writers/writer_epub.py @@ -0,0 +1,690 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import string +import StringIO +import zipfile +from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED +import urllib +import re + +## XML isn't as forgiving as HTML, so rather than generate as strings, +## use DOM to generate the XML files. +from xml.dom.minidom import parse, parseString, getDOMImplementation + +from base_writer import * +from ..htmlcleanup import stripHTML + +logger = logging.getLogger(__name__) + +class EpubWriter(BaseStoryWriter): + + @staticmethod + def getFormatName(): + return 'epub' + + @staticmethod + def getFormatExt(): + return '.epub' + + def __init__(self, config, story): + BaseStoryWriter.__init__(self, config, story) + + self.EPUB_CSS = string.Template('''${output_css}''') + + self.EPUB_TITLE_PAGE_START = string.Template(''' + + + +${title} by ${author} + + + +

    ${title} by ${authorHTML}

    +
    +''') + + self.EPUB_TITLE_ENTRY = string.Template(''' +${label}: ${value}
    +''') + + self.EPUB_NO_TITLE_ENTRY = string.Template(''' +${value}
    +''') + + self.EPUB_TITLE_PAGE_END = string.Template(''' +
    + + + +''') + + self.EPUB_TABLE_TITLE_PAGE_START = string.Template(''' + + + +${title} by ${author} + + + +

    ${title} by ${authorHTML}

    + +''') + + self.EPUB_TABLE_TITLE_ENTRY = string.Template(''' + +''') + + self.EPUB_TABLE_TITLE_WIDE_ENTRY = string.Template(''' + +''') + + self.EPUB_TABLE_NO_TITLE_ENTRY = string.Template(''' + +''') + + self.EPUB_TABLE_TITLE_PAGE_END = string.Template(''' +
    ${label}:${value}
    ${label}: ${value}
    ${label}${value}
    + + + +''') + + self.EPUB_TOC_PAGE_START = string.Template(''' + + + +${title} by ${author} + + + +
    +

    Table of Contents

    +''') + + self.EPUB_TOC_ENTRY = string.Template(''' +${chapter}
    +''') + + self.EPUB_TOC_PAGE_END = string.Template(''' +
    + + +''') + + self.EPUB_CHAPTER_START = string.Template(''' + + + +${chapter} + + + +

    ${chapter}

    +''') + + self.EPUB_CHAPTER_END = string.Template(''' + + +''') + + self.EPUB_LOG_PAGE_START = string.Template(''' + + + +Update Log + + + +

    Update Log

    +''') + + self.EPUB_LOG_UPDATE_START = string.Template(''' +

    +''') + + self.EPUB_LOG_ENTRY = string.Template(''' +${label}: ${value} +''') + + self.EPUB_LOG_UPDATE_END = string.Template(''' +

    +''') + + self.EPUB_LOG_PAGE_END = string.Template(''' + + +''') + + self.EPUB_LOG_PAGE_END = string.Template(''' + + +''') + + self.EPUB_COVER = string.Template(''' +Cover
    +cover +
    +''') + + def writeLogPage(self, out): + """ + Write the log page, but only include entries that there's + metadata for. START, ENTRY and END are expected to already be + string.Template(). START and END are expected to use the same + names as Story.metadata, but ENTRY should use id, label and value. + """ + if self.hasConfig("logpage_start"): + START = string.Template(self.getConfig("logpage_start")) + else: + START = self.EPUB_LOG_PAGE_START + + if self.hasConfig("logpage_end"): + END = string.Template(self.getConfig("logpage_end")) + else: + END = self.EPUB_LOG_PAGE_END + + # if there's a self.story.logfile, there's an existing log + # to add to. + if self.story.logfile: + logger.debug("existing logfile found, appending") + logger.debug("existing data:%s"%self._getLastLogData(self.story.logfile)) + replace_string = "" # "" + self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string)) + else: + # otherwise, write a new one. + self._write(out,START.substitute(self.story.getAllMetadata())) + self._write(out,self._makeLogEntry()) + self._write(out,END.substitute(self.story.getAllMetadata())) + + # self parsing instead of Soup because it should be simple and not + # worth the overhead. + def _getLastLogData(self,logfile): + """ + Make a dict() of the most recent(last) log entry for each piece of metadata. + Switch rindex to index to search from top instead of bottom. + """ + values = {} + for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"): + try: + # 1975-04-15 + span = ''%entry + idx = logfile.rindex(span)+len(span) + values[entry] = logfile[idx:logfile.index('',idx)] + except Exception, e: + #print("e:%s"%e) + pass + + return values + + def _makeLogEntry(self, oldvalues={}): + if self.hasConfig("logpage_update_start"): + START = string.Template(self.getConfig("logpage_update_start")) + else: + START = self.EPUB_LOG_UPDATE_START + + if self.hasConfig("logpage_entry"): + ENTRY = string.Template(self.getConfig("logpage_entry")) + else: + ENTRY = self.EPUB_LOG_ENTRY + + if self.hasConfig("logpage_update_end"): + END = string.Template(self.getConfig("logpage_update_end")) + else: + END = self.EPUB_LOG_UPDATE_END + + retval = START.substitute(self.story.getAllMetadata()) + + for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"): + if self.isValidMetaEntry(entry): + val = self.story.getMetadata(entry) + if val and ( entry not in oldvalues or val != oldvalues[entry] ): + if self.hasConfig(entry+"_label"): + label=self.getConfig(entry+"_label") + elif entry in self.titleLabels: + logger.debug("Using fallback label for %s_label"%entry) + label=self.titleLabels[entry] + else: + label="%s"%entry.title() + logger.debug("No known label for %s, fallback to '%s'"%(entry,label)) + + retval = retval + ENTRY.substitute({'id':entry, + 'label':label, + 'value':val}) + else: + # could be useful for introducing extra text, but + # mostly it makes it easy to tell when you get the + # keyword wrong. + retval = retval + entry + + retval = retval + END.substitute(self.story.getAllMetadata()) + + if self.getConfig('replace_hr'): + retval = retval.replace("
    ","
    * * *
    ") + + return retval + + def writeStoryImpl(self, out): + + ## Python 2.5 ZipFile is rather more primative than later + ## versions. It can operate on a file, or on a StringIO, but + ## not on an open stream. OTOH, I suspect we would have had + ## problems with closing and opening again to change the + ## compression type anyway. + zipio = StringIO.StringIO() + + ## mimetype must be first file and uncompressed. Python 2.5 + ## ZipFile can't change compression type file-by-file, so we + ## have to close and re-open + outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED) + outputepub.debug=3 + outputepub.writestr('mimetype','application/epub+zip') + outputepub.close() + + ## Re-open file for content. + outputepub = ZipFile(zipio, 'a', compression=ZIP_DEFLATED) + outputepub.debug=3 + + ## Create META-INF/container.xml file. The only thing it does is + ## point to content.opf + containerdom = getDOMImplementation().createDocument(None, "container", None) + containertop = containerdom.documentElement + containertop.setAttribute("version","1.0") + containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container") + rootfiles = containerdom.createElement("rootfiles") + containertop.appendChild(rootfiles) + rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":"content.opf", + "media-type":"application/oebps-package+xml"})) + outputepub.writestr("META-INF/container.xml",containerdom.toxml(encoding='utf-8')) + containerdom.unlink() + del containerdom + + ## Epub has two metadata files with real data. We're putting + ## them in content.opf (pointed to by META-INF/container.xml) + ## and toc.ncx (pointed to by content.opf) + + ## content.opf contains metadata, a 'manifest' list of all + ## other included files, and another 'spine' list of the items in the + ## file + + uniqueid= 'fanficdownloader-uid:%s-u%s-s%s' % ( + self.getMetadata('site'), + self.story.getList('authorId')[0], + self.getMetadata('storyId')) + + contentdom = getDOMImplementation().createDocument(None, "package", None) + package = contentdom.documentElement + package.setAttribute("version","2.0") + package.setAttribute("xmlns","http://www.idpf.org/2007/opf") + package.setAttribute("unique-identifier","fanficdownloader-uid") + metadata=newTag(contentdom,"metadata", + attrs={"xmlns:dc":"http://purl.org/dc/elements/1.1/", + "xmlns:opf":"http://www.idpf.org/2007/opf"}) + package.appendChild(metadata) + + metadata.appendChild(newTag(contentdom,"dc:identifier", + text=uniqueid, + attrs={"id":"fanficdownloader-uid"})) + + if self.getMetadata('title'): + metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title'))) + + if self.getMetadata('author'): + if self.story.isList('author'): + for auth in self.story.getList('author'): + metadata.appendChild(newTag(contentdom,"dc:creator", + attrs={"opf:role":"aut"}, + text=auth)) + else: + metadata.appendChild(newTag(contentdom,"dc:creator", + attrs={"opf:role":"aut"}, + text=self.getMetadata('author'))) + + metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"})) + metadata.appendChild(newTag(contentdom,"dc:rights",text="")) + if self.story.getMetadata('langcode'): + metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode'))) + else: + metadata.appendChild(newTag(contentdom,"dc:language",text='en')) + + # published, created, updated, calibre + # Leave calling self.story.getMetadataRaw directly in case date format changes. + if self.story.getMetadataRaw('datePublished'): + metadata.appendChild(newTag(contentdom,"dc:date", + attrs={"opf:event":"publication"}, + text=self.story.getMetadataRaw('datePublished').strftime("%Y-%m-%d"))) + + if self.story.getMetadataRaw('dateCreated'): + metadata.appendChild(newTag(contentdom,"dc:date", + attrs={"opf:event":"creation"}, + text=self.story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d"))) + + if self.story.getMetadataRaw('dateUpdated'): + metadata.appendChild(newTag(contentdom,"dc:date", + attrs={"opf:event":"modification"}, + text=self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%d"))) + metadata.appendChild(newTag(contentdom,"meta", + attrs={"name":"calibre:timestamp", + "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")})) + + if self.getMetadata('description'): + metadata.appendChild(newTag(contentdom,"dc:description",text= + self.getMetadata('description'))) + + for subject in self.story.getSubjectTags(): + metadata.appendChild(newTag(contentdom,"dc:subject",text=subject)) + + + if self.getMetadata('site'): + metadata.appendChild(newTag(contentdom,"dc:publisher", + text=self.getMetadata('site'))) + + if self.getMetadata('storyUrl'): + metadata.appendChild(newTag(contentdom,"dc:identifier", + attrs={"opf:scheme":"URL"}, + text=self.getMetadata('storyUrl'))) + metadata.appendChild(newTag(contentdom,"dc:source", + text=self.getMetadata('storyUrl'))) + + ## end of metadata, create manifest. + items = [] # list of (id, href, type, title) tuples(all strings) + itemrefs = [] # list of strings -- idrefs from .opfs' spines + items.append(("ncx","toc.ncx","application/x-dtbncx+xml",None)) ## we'll generate the toc.ncx file, + ## but it needs to be in the items manifest. + + guide = None + coverIO = None + + coverimgid = "image0000" + if not self.story.cover and self.story.oldcover: + logger.debug("writer_epub: no new cover, has old cover, write image.") + (oldcoverhtmlhref, + oldcoverhtmltype, + oldcoverhtmldata, + oldcoverimghref, + oldcoverimgtype, + oldcoverimgdata) = self.story.oldcover + outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata) + outputepub.writestr(oldcoverimghref,oldcoverimgdata) + + coverimgid = "image0" + items.append((coverimgid, + oldcoverimghref, + oldcoverimgtype, + None)) + items.append(("cover",oldcoverhtmlhref,oldcoverhtmltype,None)) + itemrefs.append("cover") + metadata.appendChild(newTag(contentdom,"meta",{"content":"image0", + "name":"cover"})) + guide = newTag(contentdom,"guide") + guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover", + "title":"Cover", + "href":oldcoverhtmlhref})) + + + + if self.getConfig('include_images'): + imgcount=0 + for imgmap in self.story.getImgUrls(): + imgfile = "OEBPS/"+imgmap['newsrc'] + outputepub.writestr(imgfile,imgmap['data']) + items.append(("image%04d"%imgcount, + imgfile, + imgmap['mime'], + None)) + imgcount+=1 + if 'cover' in imgfile: + # make sure coverimgid is set to the cover, not + # just the first image. + coverimgid = items[-1][0] + + + items.append(("style","OEBPS/stylesheet.css","text/css",None)) + + if self.story.cover: + # Note that the id of the cover xhmtl *must* be 'cover' + # for it to work on Nook. + items.append(("cover","OEBPS/cover.xhtml","application/xhtml+xml",None)) + itemrefs.append("cover") + # + # + metadata.appendChild(newTag(contentdom,"meta",{"content":coverimgid, + "name":"cover"})) + # cover stuff for later: + # at end of : + # + # + # + guide = newTag(contentdom,"guide") + guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover", + "title":"Cover", + "href":"OEBPS/cover.xhtml"})) + + if self.hasConfig("cover_content"): + COVER = string.Template(self.getConfig("cover_content")) + else: + COVER = self.EPUB_COVER + coverIO = StringIO.StringIO() + coverIO.write(COVER.substitute(dict(self.story.getAllMetadata().items()+{'coverimg':self.story.cover}.items()))) + + if self.getConfig("include_titlepage"): + items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page")) + itemrefs.append("title_page") + if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly : + items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents")) + itemrefs.append("toc_page") + + dologpage = ( self.getConfig("include_logpage") == "smart" and \ + (self.story.logfile or self.story.getMetadataRaw("status") == "In-Progress") ) \ + or self.getConfig("include_logpage") == "true" + + if dologpage: + items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log")) + itemrefs.append("log_page") + + for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)): + if html: + i=index+1 + items.append(("file%04d"%i, + "OEBPS/file%04d.xhtml"%i, + "application/xhtml+xml", + title)) + itemrefs.append("file%04d"%i) + + manifest = contentdom.createElement("manifest") + package.appendChild(manifest) + for item in items: + (id,href,type,title)=item + manifest.appendChild(newTag(contentdom,"item", + attrs={'id':id, + 'href':href, + 'media-type':type})) + + spine = newTag(contentdom,"spine",attrs={"toc":"ncx"}) + package.appendChild(spine) + for itemref in itemrefs: + spine.appendChild(newTag(contentdom,"itemref", + attrs={"idref":itemref, + "linear":"yes"})) + # guide only exists if there's a cover. + if guide: + package.appendChild(guide) + + # write content.opf to zip. + contentxml = contentdom.toxml(encoding='utf-8') + + # tweak for brain damaged Nook STR. Nook insists on name before content. + contentxml = contentxml.replace(''%coverimgid, + ''%coverimgid) + outputepub.writestr("content.opf",contentxml) + + contentdom.unlink() + del contentdom + + ## create toc.ncx file + tocncxdom = getDOMImplementation().createDocument(None, "ncx", None) + ncx = tocncxdom.documentElement + ncx.setAttribute("version","2005-1") + ncx.setAttribute("xmlns","http://www.daisy.org/z3986/2005/ncx/") + head = tocncxdom.createElement("head") + ncx.appendChild(head) + head.appendChild(newTag(tocncxdom,"meta", + attrs={"name":"dtb:uid", "content":uniqueid})) + head.appendChild(newTag(tocncxdom,"meta", + attrs={"name":"dtb:depth", "content":"1"})) + head.appendChild(newTag(tocncxdom,"meta", + attrs={"name":"dtb:totalPageCount", "content":"0"})) + head.appendChild(newTag(tocncxdom,"meta", + attrs={"name":"dtb:maxPageNumber", "content":"0"})) + + docTitle = tocncxdom.createElement("docTitle") + docTitle.appendChild(newTag(tocncxdom,"text",text=self.getMetadata('title'))) + ncx.appendChild(docTitle) + + tocnavMap = tocncxdom.createElement("navMap") + ncx.appendChild(tocnavMap) + + # + # + # + # + # + # + index=0 + for item in items: + (id,href,type,title)=item + # only items to be skipped, cover.xhtml, images, toc.ncx, stylesheet.css, should have no title. + if title : + navPoint = newTag(tocncxdom,"navPoint", + attrs={'id':id, + 'playOrder':str(index)}) + tocnavMap.appendChild(navPoint) + navLabel = newTag(tocncxdom,"navLabel") + navPoint.appendChild(navLabel) + ## the xml library will re-escape as needed. + navLabel.appendChild(newTag(tocncxdom,"text",text=stripHTML(title))) + navPoint.appendChild(newTag(tocncxdom,"content",attrs={"src":href})) + index=index+1 + + # write toc.ncx to zip file + outputepub.writestr("toc.ncx",tocncxdom.toxml(encoding='utf-8')) + tocncxdom.unlink() + del tocncxdom + + # write stylesheet.css file. + outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata())) + + # write title page. + if self.getConfig("titlepage_use_table"): + TITLE_PAGE_START = self.EPUB_TABLE_TITLE_PAGE_START + TITLE_ENTRY = self.EPUB_TABLE_TITLE_ENTRY + WIDE_TITLE_ENTRY = self.EPUB_TABLE_TITLE_WIDE_ENTRY + NO_TITLE_ENTRY = self.EPUB_TABLE_NO_TITLE_ENTRY + TITLE_PAGE_END = self.EPUB_TABLE_TITLE_PAGE_END + else: + TITLE_PAGE_START = self.EPUB_TITLE_PAGE_START + TITLE_ENTRY = self.EPUB_TITLE_ENTRY + WIDE_TITLE_ENTRY = self.EPUB_TITLE_ENTRY # same, only wide in tables. + NO_TITLE_ENTRY = self.EPUB_NO_TITLE_ENTRY + TITLE_PAGE_END = self.EPUB_TITLE_PAGE_END + + if coverIO: + outputepub.writestr("OEBPS/cover.xhtml",coverIO.getvalue()) + coverIO.close() + + titlepageIO = StringIO.StringIO() + self.writeTitlePage(out=titlepageIO, + START=TITLE_PAGE_START, + ENTRY=TITLE_ENTRY, + WIDE_ENTRY=WIDE_TITLE_ENTRY, + END=TITLE_PAGE_END, + NO_TITLE_ENTRY=NO_TITLE_ENTRY) + if titlepageIO.getvalue(): # will be false if no title page. + outputepub.writestr("OEBPS/title_page.xhtml",titlepageIO.getvalue()) + titlepageIO.close() + + # write toc page. + tocpageIO = StringIO.StringIO() + self.writeTOCPage(tocpageIO, + self.EPUB_TOC_PAGE_START, + self.EPUB_TOC_ENTRY, + self.EPUB_TOC_PAGE_END) + if tocpageIO.getvalue(): # will be false if no toc page. + outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue()) + tocpageIO.close() + + if dologpage: + # write log page. + logpageIO = StringIO.StringIO() + self.writeLogPage(logpageIO) + outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue()) + logpageIO.close() + + if self.hasConfig('chapter_start'): + CHAPTER_START = string.Template(self.getConfig("chapter_start")) + else: + CHAPTER_START = self.EPUB_CHAPTER_START + + if self.hasConfig('chapter_end'): + CHAPTER_END = string.Template(self.getConfig("chapter_end")) + else: + CHAPTER_END = self.EPUB_CHAPTER_END + + for index, (url,title,html) in enumerate(self.story.getChapters()): + if html: + logger.debug('Writing chapter text for: %s' % title) + vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals) + # ffnet(& maybe others) gives the whole chapter text + # as one line. This causes problems for nook(at + # least) when the chapter size starts getting big + # (200k+) + #fullhtml = fullhtml.replace('

    ','

    \n').replace('
    ','
    \n') + # The replaces above added tons of extra newlines + # during *each* epub update. The regexp version adds + # only one and removes any extra. + fullhtml = re.sub(r'(

    |
    )\n*',r'\1\n',fullhtml) + + outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8')) + del fullhtml + + if self.story.calibrebookmark: + outputepub.writestr("META-INF/calibre_bookmarks.txt",self.story.calibrebookmark) + + # declares all the files created by Windows. otherwise, when + # it runs in appengine, windows unzips the files as 000 perms. + for zf in outputepub.filelist: + zf.create_system = 0 + outputepub.close() + out.write(zipio.getvalue()) + zipio.close() + +## Utility method for creating new tags. +def newTag(dom,name,attrs=None,text=None): + tag = dom.createElement(name) + if( attrs is not None ): + for attr in attrs.keys(): + tag.setAttribute(attr,attrs[attr]) + if( text is not None ): + tag.appendChild(dom.createTextNode(text)) + return tag + diff --git a/fanficdownloader/writers/writer_html.py b/fanficdownloader/writers/writer_html.py new file mode 100644 index 0000000..41b0175 --- /dev/null +++ b/fanficdownloader/writers/writer_html.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import string + +from base_writer import * + +class HTMLWriter(BaseStoryWriter): + + @staticmethod + def getFormatName(): + return 'html' + + @staticmethod + def getFormatExt(): + return '.html' + + def __init__(self, config, story): + BaseStoryWriter.__init__(self, config, story) + + self.HTML_FILE_START = string.Template(''' + + + +${title} by ${author} + + + +

    ${title} by ${authorHTML}

    +''') + + self.HTML_COVER = string.Template(''' +cover +''') + + self.HTML_TITLE_PAGE_START = string.Template(''' + +''') + + self.HTML_TITLE_ENTRY = string.Template(''' + +''') + + self.HTML_TITLE_PAGE_END = string.Template(''' +
    ${label}:${value}
    +''') + + self.HTML_TOC_PAGE_START = string.Template(''' +

    Table of Contents

    +

    +''') + + self.HTML_TOC_ENTRY = string.Template(''' +${chapter}
    +''') + + self.HTML_TOC_PAGE_END = string.Template(''' +

    +''') + + self.HTML_CHAPTER_START = string.Template(''' +

    ${chapter}

     +''') + + self.HTML_CHAPTER_END = string.Template('') + + self.HTML_FILE_END = string.Template(''' + +''') + + + def writeStoryImpl(self, out): + + if self.hasConfig("cover_content"): + COVER = string.Template(self.getConfig("cover_content")) + else: + COVER = self.HTML_COVER + + if self.hasConfig('file_start'): + FILE_START = string.Template(self.getConfig("file_start")) + else: + FILE_START = self.HTML_FILE_START + + if self.hasConfig('file_end'): + FILE_END = string.Template(self.getConfig("file_end")) + else: + FILE_END = self.HTML_FILE_END + + self._write(out,FILE_START.substitute(self.story.getAllMetadata())) + + if self.getConfig('include_images') and self.story.cover: + self._write(out,COVER.substitute(dict(self.story.getAllMetadata().items()+{'coverimg':self.story.cover}.items()))) + + self.writeTitlePage(out, + self.HTML_TITLE_PAGE_START, + self.HTML_TITLE_ENTRY, + self.HTML_TITLE_PAGE_END) + + self.writeTOCPage(out, + self.HTML_TOC_PAGE_START, + self.HTML_TOC_ENTRY, + self.HTML_TOC_PAGE_END) + + if self.hasConfig('chapter_start'): + CHAPTER_START = string.Template(self.getConfig("chapter_start")) + else: + CHAPTER_START = self.HTML_CHAPTER_START + + if self.hasConfig('chapter_end'): + CHAPTER_END = string.Template(self.getConfig("chapter_end")) + else: + CHAPTER_END = self.HTML_CHAPTER_END + + for index, (url,title,html) in enumerate(self.story.getChapters()): + if html: + logging.debug('Writing chapter text for: %s' % title) + vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + self._write(out,CHAPTER_START.substitute(vals)) + self._write(out,html) + self._write(out,CHAPTER_END.substitute(vals)) + + self._write(out,FILE_END.substitute(self.story.getAllMetadata())) + + if self.getConfig('include_images'): + for imgmap in self.story.getImgUrls(): + self.writeFile(imgmap['newsrc'],imgmap['data']) + diff --git a/fanficdownloader/writers/writer_mobi.py b/fanficdownloader/writers/writer_mobi.py new file mode 100644 index 0000000..4396ab0 --- /dev/null +++ b/fanficdownloader/writers/writer_mobi.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import string +import StringIO + +from base_writer import * +from ..htmlcleanup import stripHTML +from ..mobi import Converter +from ..exceptions import FailedToWriteOutput + +logger = logging.getLogger(__name__) + +class MobiWriter(BaseStoryWriter): + + @staticmethod + def getFormatName(): + return 'mobi' + + @staticmethod + def getFormatExt(): + return '.mobi' + + def __init__(self, config, story): + BaseStoryWriter.__init__(self, config, story) + + self.MOBI_TITLE_PAGE_START = string.Template(''' + + + +${title} by ${author} + + +

    ${title} by ${authorHTML}

    +
    +''') + + self.MOBI_TITLE_ENTRY = string.Template(''' +${label}: ${value}
    +''') + + self.MOBI_NO_TITLE_ENTRY = string.Template(''' +${value}
    +''') + + self.MOBI_TITLE_PAGE_END = string.Template(''' +
    + + + +''') + + self.MOBI_TABLE_TITLE_PAGE_START = string.Template(''' + + + +${title} by ${author} + + +

    ${title} by ${authorHTML}

    + +''') + + self.MOBI_TABLE_TITLE_ENTRY = string.Template(''' + +''') + + self.MOBI_TABLE_TITLE_WIDE_ENTRY = string.Template(''' + +''') + + self.MOBI_TABLE_NO_TITLE_WIDE_ENTRY = string.Template(''' + +''') + + self.MOBI_TABLE_TITLE_PAGE_END = string.Template(''' +
    ${label}:${value}
    ${label}: ${value}
    ${value}
    + + + +''') + + self.MOBI_CHAPTER_START = string.Template(''' + + + +${chapter} + + +

    ${chapter}

    +''') + + self.MOBI_CHAPTER_END = string.Template(''' + + +''') + + def writeStoryImpl(self, out): + + files = [] + + # write title page. + if self.getConfig("titlepage_use_table"): + TITLE_PAGE_START = self.MOBI_TABLE_TITLE_PAGE_START + TITLE_ENTRY = self.MOBI_TABLE_TITLE_ENTRY + WIDE_TITLE_ENTRY = self.MOBI_TABLE_TITLE_WIDE_ENTRY + NO_TITLE_ENTRY = self.MOBI_TABLE_NO_TITLE_ENTRY + TITLE_PAGE_END = self.MOBI_TABLE_TITLE_PAGE_END + else: + TITLE_PAGE_START = self.MOBI_TITLE_PAGE_START + TITLE_ENTRY = self.MOBI_TITLE_ENTRY + WIDE_TITLE_ENTRY = self.MOBI_TITLE_ENTRY # same, only wide in tables. + NO_TITLE_ENTRY = self.MOBI_NO_TITLE_ENTRY + TITLE_PAGE_END = self.MOBI_TITLE_PAGE_END + + titlepageIO = StringIO.StringIO() + self.writeTitlePage(out=titlepageIO, + START=TITLE_PAGE_START, + ENTRY=TITLE_ENTRY, + WIDE_ENTRY=WIDE_TITLE_ENTRY, + END=TITLE_PAGE_END, + NO_TITLE_ENTRY=NO_TITLE_ENTRY) + if titlepageIO.getvalue(): # will be false if no title page. + files.append(titlepageIO.getvalue()) + titlepageIO.close() + + ## MOBI always has a TOC injected by mobi.py because there's + ## no meta-data TOC. + # # write toc page. + # tocpageIO = StringIO.StringIO() + # self.writeTOCPage(tocpageIO, + # self.MOBI_TOC_PAGE_START, + # self.MOBI_TOC_ENTRY, + # self.MOBI_TOC_PAGE_END) + # if tocpageIO.getvalue(): # will be false if no toc page. + # files.append(tocpageIO.getvalue()) + # tocpageIO.close() + + if self.hasConfig('chapter_start'): + CHAPTER_START = string.Template(self.getConfig("chapter_start")) + else: + CHAPTER_START = self.MOBI_CHAPTER_START + + if self.hasConfig('chapter_end'): + CHAPTER_END = string.Template(self.getConfig("chapter_end")) + else: + CHAPTER_END = self.MOBI_CHAPTER_END + + for index, (url,title,html) in enumerate(self.story.getChapters()): + if html: + logger.debug('Writing chapter text for: %s' % title) + vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals) + # ffnet(& maybe others) gives the whole chapter text + # as one line. This causes problems for nook(at + # least) when the chapter size starts getting big + # (200k+) + fullhtml = fullhtml.replace('

    ','

    \n').replace('
    ','
    \n') + files.append(fullhtml.encode('utf-8')) + del fullhtml + + c = Converter(title=self.getMetadata('title'), + author=self.getMetadata('author'), + publisher=self.getMetadata('site')) + mobidata = c.ConvertStrings(files) + if len(mobidata) < 1: + raise FailedToWriteOutput("Zero length mobi output") + out.write(mobidata) + + del files + del mobidata + +## Utility method for creating new tags. +def newTag(dom,name,attrs=None,text=None): + tag = dom.createElement(name) + if( attrs is not None ): + for attr in attrs.keys(): + tag.setAttribute(attr,attrs[attr]) + if( text is not None ): + tag.appendChild(dom.createTextNode(text)) + return tag + diff --git a/fanficdownloader/writers/writer_txt.py b/fanficdownloader/writers/writer_txt.py new file mode 100644 index 0000000..8705a41 --- /dev/null +++ b/fanficdownloader/writers/writer_txt.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- + +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import string +from textwrap import wrap + +from base_writer import * + +from ..html2text import html2text + +## In BaseStoryWriter, we define _write to encode objects +## back into for true output. But txt needs to write the +## title page and TOC to a buffer first to wordwrap. And StringIO +## gets pissy about unicode bytes in its buflist. This decodes the +## unicode containing object passed in back to a +## object so they join up properly. Could override _write to not +## encode and do out.write(whatever.encode('utf8') instead. Honestly +## not sure which is uglier. +class KludgeStringIO(): + def __init__(self, buf = ''): + self.buflist=[] + def write(self,s): + try: + s=s.decode('utf-8') + except: + pass + self.buflist.append(s) + def getvalue(self): + return u''.join(self.buflist) + def close(self): + pass + +class TextWriter(BaseStoryWriter): + + @staticmethod + def getFormatName(): + return 'txt' + + @staticmethod + def getFormatExt(): + return '.txt' + + def __init__(self, config, story): + + BaseStoryWriter.__init__(self, config, story) + + self.TEXT_FILE_START = string.Template(u''' + + +${title} + +by ${author} + + +''') + + self.TEXT_TITLE_PAGE_START = string.Template(u''' +''') + + self.TEXT_TITLE_ENTRY = string.Template(u'''${label}: ${value} +''') + + self.TEXT_TITLE_PAGE_END = string.Template(u''' + + +''') + + self.TEXT_TOC_PAGE_START = string.Template(u''' + +TABLE OF CONTENTS + +''') + + self.TEXT_TOC_ENTRY = string.Template(u''' +${chapter} +''') + + self.TEXT_TOC_PAGE_END = string.Template(u''' +''') + + self.TEXT_CHAPTER_START = string.Template(u''' + +\t${chapter} + +''') + self.TEXT_CHAPTER_END = string.Template(u'') + + self.TEXT_FILE_END = string.Template(u''' + +End file. +''') + + def writeStoryImpl(self, out): + + self.wrap_width = self.getConfig('wrap_width') + if self.wrap_width == '' or self.wrap_width == '0': + self.wrap_width = None + else: + self.wrap_width = int(self.wrap_width) + + wrapout = KludgeStringIO() + + if self.hasConfig("file_start"): + FILE_START = string.Template(self.getConfig("file_start")) + else: + FILE_START = self.TEXT_FILE_START + + if self.hasConfig("file_end"): + FILE_END = string.Template(self.getConfig("file_end")) + else: + FILE_END = self.TEXT_FILE_END + + wrapout.write(FILE_START.substitute(self.story.getAllMetadata())) + + self.writeTitlePage(wrapout, + self.TEXT_TITLE_PAGE_START, + self.TEXT_TITLE_ENTRY, + self.TEXT_TITLE_PAGE_END) + towrap = wrapout.getvalue() + + self.writeTOCPage(wrapout, + self.TEXT_TOC_PAGE_START, + self.TEXT_TOC_ENTRY, + self.TEXT_TOC_PAGE_END) + + towrap = wrapout.getvalue() + wrapout.close() + towrap = removeAllEntities(towrap) + + self._write(out,self.lineends(self.wraplines(towrap))) + + if self.hasConfig('chapter_start'): + CHAPTER_START = string.Template(self.getConfig("chapter_start")) + else: + CHAPTER_START = self.TEXT_CHAPTER_START + + if self.hasConfig('chapter_end'): + CHAPTER_END = string.Template(self.getConfig("chapter_end")) + else: + CHAPTER_END = self.TEXT_CHAPTER_END + + for index, (url, title,html) in enumerate(self.story.getChapters()): + if html: + logging.debug('Writing chapter text for: %s' % title) + vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals))))) + self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width))) + self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals))))) + + self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata())))) + + def wraplines(self, text): + + if not self.wrap_width: + return text + + result='' + for para in text.split("\n"): + first=True + for line in wrap(para, self.wrap_width): + if first: + first=False + else: + result += u"\n" + result += line + result += u"\n" + return result + + ## The appengine will return unix line endings. + def lineends(self, txt): + txt = txt.replace('\r','') + if self.getConfig("windows_eol"): + txt = txt.replace('\n',u'\r\n') + return txt + diff --git a/ffstorage.py b/ffstorage.py new file mode 100644 index 0000000..bad9b4a --- /dev/null +++ b/ffstorage.py @@ -0,0 +1,63 @@ +# Copyright 2011 Fanficdownloader team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pickle, copy +from google.appengine.ext import db + +class ObjectProperty(db.Property): + data_type = db.Blob + + def get_value_for_datastore(self, model_instance): + value = self.__get__(model_instance, model_instance.__class__) + pickled_val = pickle.dumps(value,protocol=pickle.HIGHEST_PROTOCOL) + if value is not None: return db.Blob(pickled_val) + + def make_value_from_datastore(self, value): + if value is not None: return pickle.loads(value) + + def default_value(self): + return copy.copy(self.default) + +class DownloadMeta(db.Model): + user = db.UserProperty() + url = db.StringProperty() + name = db.StringProperty() + title = db.StringProperty() + author = db.StringProperty() + format = db.StringProperty() + failure = db.TextProperty() + completed = db.BooleanProperty(default=False) + date = db.DateTimeProperty(auto_now_add=True) + version = db.StringProperty() + # data_chunks is implicit from DownloadData def. + +class DownloadData(db.Model): + download = db.ReferenceProperty(DownloadMeta, + collection_name='data_chunks') + blob = db.BlobProperty() + index = db.IntegerProperty() + +class UserConfig(db.Model): + user = db.UserProperty() + config = db.BlobProperty() + +class SavedMeta(db.Model): + url = db.StringProperty() + title = db.StringProperty() + author = db.StringProperty() + date = db.DateTimeProperty(auto_now_add=True) + count = db.IntegerProperty() + meta = ObjectProperty() + diff --git a/html5lib/LICENSE b/html5lib/LICENSE new file mode 100644 index 0000000..c87fa7a --- /dev/null +++ b/html5lib/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2006-2013 James Graham and other contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/html5lib/__init__.py b/html5lib/__init__.py new file mode 100644 index 0000000..19a4b7d --- /dev/null +++ b/html5lib/__init__.py @@ -0,0 +1,23 @@ +""" +HTML parsing library based on the WHATWG "HTML5" +specification. The parser is designed to be compatible with existing +HTML found in the wild and implements well-defined error recovery that +is largely compatible with modern desktop web browsers. + +Example usage: + +import html5lib +f = open("my_document.html") +tree = html5lib.parse(f) +""" + +from __future__ import absolute_import, division, unicode_literals + +from .html5parser import HTMLParser, parse, parseFragment +from .treebuilders import getTreeBuilder +from .treewalkers import getTreeWalker +from .serializer import serialize + +__all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder", + "getTreeWalker", "serialize"] +__version__ = "0.999" diff --git a/html5lib/constants.py b/html5lib/constants.py new file mode 100644 index 0000000..e708984 --- /dev/null +++ b/html5lib/constants.py @@ -0,0 +1,3104 @@ +from __future__ import absolute_import, division, unicode_literals + +import string +import gettext +_ = gettext.gettext + +EOF = None + +E = { + "null-character": + _("Null character in input stream, replaced with U+FFFD."), + "invalid-codepoint": + _("Invalid codepoint in stream."), + "incorrectly-placed-solidus": + _("Solidus (/) incorrectly placed in tag."), + "incorrect-cr-newline-entity": + _("Incorrect CR newline entity, replaced with LF."), + "illegal-windows-1252-entity": + _("Entity used with illegal number (windows-1252 reference)."), + "cant-convert-numeric-entity": + _("Numeric entity couldn't be converted to character " + "(codepoint U+%(charAsInt)08x)."), + "illegal-codepoint-for-numeric-entity": + _("Numeric entity represents an illegal codepoint: " + "U+%(charAsInt)08x."), + "numeric-entity-without-semicolon": + _("Numeric entity didn't end with ';'."), + "expected-numeric-entity-but-got-eof": + _("Numeric entity expected. Got end of file instead."), + "expected-numeric-entity": + _("Numeric entity expected but none found."), + "named-entity-without-semicolon": + _("Named entity didn't end with ';'."), + "expected-named-entity": + _("Named entity expected. Got none."), + "attributes-in-end-tag": + _("End tag contains unexpected attributes."), + 'self-closing-flag-on-end-tag': + _("End tag contains unexpected self-closing flag."), + "expected-tag-name-but-got-right-bracket": + _("Expected tag name. Got '>' instead."), + "expected-tag-name-but-got-question-mark": + _("Expected tag name. Got '?' instead. (HTML doesn't " + "support processing instructions.)"), + "expected-tag-name": + _("Expected tag name. Got something else instead"), + "expected-closing-tag-but-got-right-bracket": + _("Expected closing tag. Got '>' instead. Ignoring ''."), + "expected-closing-tag-but-got-eof": + _("Expected closing tag. Unexpected end of file."), + "expected-closing-tag-but-got-char": + _("Expected closing tag. Unexpected character '%(data)s' found."), + "eof-in-tag-name": + _("Unexpected end of file in the tag name."), + "expected-attribute-name-but-got-eof": + _("Unexpected end of file. Expected attribute name instead."), + "eof-in-attribute-name": + _("Unexpected end of file in attribute name."), + "invalid-character-in-attribute-name": + _("Invalid character in attribute name"), + "duplicate-attribute": + _("Dropped duplicate attribute on tag."), + "expected-end-of-tag-name-but-got-eof": + _("Unexpected end of file. Expected = or end of tag."), + "expected-attribute-value-but-got-eof": + _("Unexpected end of file. Expected attribute value."), + "expected-attribute-value-but-got-right-bracket": + _("Expected attribute value. Got '>' instead."), + 'equals-in-unquoted-attribute-value': + _("Unexpected = in unquoted attribute"), + 'unexpected-character-in-unquoted-attribute-value': + _("Unexpected character in unquoted attribute"), + "invalid-character-after-attribute-name": + _("Unexpected character after attribute name."), + "unexpected-character-after-attribute-value": + _("Unexpected character after attribute value."), + "eof-in-attribute-value-double-quote": + _("Unexpected end of file in attribute value (\")."), + "eof-in-attribute-value-single-quote": + _("Unexpected end of file in attribute value (')."), + "eof-in-attribute-value-no-quotes": + _("Unexpected end of file in attribute value."), + "unexpected-EOF-after-solidus-in-tag": + _("Unexpected end of file in tag. Expected >"), + "unexpected-character-after-solidus-in-tag": + _("Unexpected character after / in tag. Expected >"), + "expected-dashes-or-doctype": + _("Expected '--' or 'DOCTYPE'. Not found."), + "unexpected-bang-after-double-dash-in-comment": + _("Unexpected ! after -- in comment"), + "unexpected-space-after-double-dash-in-comment": + _("Unexpected space after -- in comment"), + "incorrect-comment": + _("Incorrect comment."), + "eof-in-comment": + _("Unexpected end of file in comment."), + "eof-in-comment-end-dash": + _("Unexpected end of file in comment (-)"), + "unexpected-dash-after-double-dash-in-comment": + _("Unexpected '-' after '--' found in comment."), + "eof-in-comment-double-dash": + _("Unexpected end of file in comment (--)."), + "eof-in-comment-end-space-state": + _("Unexpected end of file in comment."), + "eof-in-comment-end-bang-state": + _("Unexpected end of file in comment."), + "unexpected-char-in-comment": + _("Unexpected character in comment found."), + "need-space-after-doctype": + _("No space after literal string 'DOCTYPE'."), + "expected-doctype-name-but-got-right-bracket": + _("Unexpected > character. Expected DOCTYPE name."), + "expected-doctype-name-but-got-eof": + _("Unexpected end of file. Expected DOCTYPE name."), + "eof-in-doctype-name": + _("Unexpected end of file in DOCTYPE name."), + "eof-in-doctype": + _("Unexpected end of file in DOCTYPE."), + "expected-space-or-right-bracket-in-doctype": + _("Expected space or '>'. Got '%(data)s'"), + "unexpected-end-of-doctype": + _("Unexpected end of DOCTYPE."), + "unexpected-char-in-doctype": + _("Unexpected character in DOCTYPE."), + "eof-in-innerhtml": + _("XXX innerHTML EOF"), + "unexpected-doctype": + _("Unexpected DOCTYPE. Ignored."), + "non-html-root": + _("html needs to be the first start tag."), + "expected-doctype-but-got-eof": + _("Unexpected End of file. Expected DOCTYPE."), + "unknown-doctype": + _("Erroneous DOCTYPE."), + "expected-doctype-but-got-chars": + _("Unexpected non-space characters. Expected DOCTYPE."), + "expected-doctype-but-got-start-tag": + _("Unexpected start tag (%(name)s). Expected DOCTYPE."), + "expected-doctype-but-got-end-tag": + _("Unexpected end tag (%(name)s). Expected DOCTYPE."), + "end-tag-after-implied-root": + _("Unexpected end tag (%(name)s) after the (implied) root element."), + "expected-named-closing-tag-but-got-eof": + _("Unexpected end of file. Expected end tag (%(name)s)."), + "two-heads-are-not-better-than-one": + _("Unexpected start tag head in existing head. Ignored."), + "unexpected-end-tag": + _("Unexpected end tag (%(name)s). Ignored."), + "unexpected-start-tag-out-of-my-head": + _("Unexpected start tag (%(name)s) that can be in head. Moved."), + "unexpected-start-tag": + _("Unexpected start tag (%(name)s)."), + "missing-end-tag": + _("Missing end tag (%(name)s)."), + "missing-end-tags": + _("Missing end tags (%(name)s)."), + "unexpected-start-tag-implies-end-tag": + _("Unexpected start tag (%(startName)s) " + "implies end tag (%(endName)s)."), + "unexpected-start-tag-treated-as": + _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."), + "deprecated-tag": + _("Unexpected start tag %(name)s. Don't use it!"), + "unexpected-start-tag-ignored": + _("Unexpected start tag %(name)s. Ignored."), + "expected-one-end-tag-but-got-another": + _("Unexpected end tag (%(gotName)s). " + "Missing end tag (%(expectedName)s)."), + "end-tag-too-early": + _("End tag (%(name)s) seen too early. Expected other end tag."), + "end-tag-too-early-named": + _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."), + "end-tag-too-early-ignored": + _("End tag (%(name)s) seen too early. Ignored."), + "adoption-agency-1.1": + _("End tag (%(name)s) violates step 1, " + "paragraph 1 of the adoption agency algorithm."), + "adoption-agency-1.2": + _("End tag (%(name)s) violates step 1, " + "paragraph 2 of the adoption agency algorithm."), + "adoption-agency-1.3": + _("End tag (%(name)s) violates step 1, " + "paragraph 3 of the adoption agency algorithm."), + "adoption-agency-4.4": + _("End tag (%(name)s) violates step 4, " + "paragraph 4 of the adoption agency algorithm."), + "unexpected-end-tag-treated-as": + _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."), + "no-end-tag": + _("This element (%(name)s) has no end tag."), + "unexpected-implied-end-tag-in-table": + _("Unexpected implied end tag (%(name)s) in the table phase."), + "unexpected-implied-end-tag-in-table-body": + _("Unexpected implied end tag (%(name)s) in the table body phase."), + "unexpected-char-implies-table-voodoo": + _("Unexpected non-space characters in " + "table context caused voodoo mode."), + "unexpected-hidden-input-in-table": + _("Unexpected input with type hidden in table context."), + "unexpected-form-in-table": + _("Unexpected form in table context."), + "unexpected-start-tag-implies-table-voodoo": + _("Unexpected start tag (%(name)s) in " + "table context caused voodoo mode."), + "unexpected-end-tag-implies-table-voodoo": + _("Unexpected end tag (%(name)s) in " + "table context caused voodoo mode."), + "unexpected-cell-in-table-body": + _("Unexpected table cell start tag (%(name)s) " + "in the table body phase."), + "unexpected-cell-end-tag": + _("Got table cell end tag (%(name)s) " + "while required end tags are missing."), + "unexpected-end-tag-in-table-body": + _("Unexpected end tag (%(name)s) in the table body phase. Ignored."), + "unexpected-implied-end-tag-in-table-row": + _("Unexpected implied end tag (%(name)s) in the table row phase."), + "unexpected-end-tag-in-table-row": + _("Unexpected end tag (%(name)s) in the table row phase. Ignored."), + "unexpected-select-in-select": + _("Unexpected select start tag in the select phase " + "treated as select end tag."), + "unexpected-input-in-select": + _("Unexpected input start tag in the select phase."), + "unexpected-start-tag-in-select": + _("Unexpected start tag token (%(name)s in the select phase. " + "Ignored."), + "unexpected-end-tag-in-select": + _("Unexpected end tag (%(name)s) in the select phase. Ignored."), + "unexpected-table-element-start-tag-in-select-in-table": + _("Unexpected table element start tag (%(name)s) in the select in table phase."), + "unexpected-table-element-end-tag-in-select-in-table": + _("Unexpected table element end tag (%(name)s) in the select in table phase."), + "unexpected-char-after-body": + _("Unexpected non-space characters in the after body phase."), + "unexpected-start-tag-after-body": + _("Unexpected start tag token (%(name)s)" + " in the after body phase."), + "unexpected-end-tag-after-body": + _("Unexpected end tag token (%(name)s)" + " in the after body phase."), + "unexpected-char-in-frameset": + _("Unexpected characters in the frameset phase. Characters ignored."), + "unexpected-start-tag-in-frameset": + _("Unexpected start tag token (%(name)s)" + " in the frameset phase. Ignored."), + "unexpected-frameset-in-frameset-innerhtml": + _("Unexpected end tag token (frameset) " + "in the frameset phase (innerHTML)."), + "unexpected-end-tag-in-frameset": + _("Unexpected end tag token (%(name)s)" + " in the frameset phase. Ignored."), + "unexpected-char-after-frameset": + _("Unexpected non-space characters in the " + "after frameset phase. Ignored."), + "unexpected-start-tag-after-frameset": + _("Unexpected start tag (%(name)s)" + " in the after frameset phase. Ignored."), + "unexpected-end-tag-after-frameset": + _("Unexpected end tag (%(name)s)" + " in the after frameset phase. Ignored."), + "unexpected-end-tag-after-body-innerhtml": + _("Unexpected end tag after body(innerHtml)"), + "expected-eof-but-got-char": + _("Unexpected non-space characters. Expected end of file."), + "expected-eof-but-got-start-tag": + _("Unexpected start tag (%(name)s)" + ". Expected end of file."), + "expected-eof-but-got-end-tag": + _("Unexpected end tag (%(name)s)" + ". Expected end of file."), + "eof-in-table": + _("Unexpected end of file. Expected table content."), + "eof-in-select": + _("Unexpected end of file. Expected select content."), + "eof-in-frameset": + _("Unexpected end of file. Expected frameset content."), + "eof-in-script-in-script": + _("Unexpected end of file. Expected script content."), + "eof-in-foreign-lands": + _("Unexpected end of file. Expected foreign content"), + "non-void-element-with-trailing-solidus": + _("Trailing solidus not allowed on element %(name)s"), + "unexpected-html-element-in-foreign-content": + _("Element %(name)s not allowed in a non-html context"), + "unexpected-end-tag-before-html": + _("Unexpected end tag (%(name)s) before html."), + "XXX-undefined-error": + _("Undefined error (this sucks and should be fixed)"), +} + +namespaces = { + "html": "http://www.w3.org/1999/xhtml", + "mathml": "http://www.w3.org/1998/Math/MathML", + "svg": "http://www.w3.org/2000/svg", + "xlink": "http://www.w3.org/1999/xlink", + "xml": "http://www.w3.org/XML/1998/namespace", + "xmlns": "http://www.w3.org/2000/xmlns/" +} + +scopingElements = frozenset(( + (namespaces["html"], "applet"), + (namespaces["html"], "caption"), + (namespaces["html"], "html"), + (namespaces["html"], "marquee"), + (namespaces["html"], "object"), + (namespaces["html"], "table"), + (namespaces["html"], "td"), + (namespaces["html"], "th"), + (namespaces["mathml"], "mi"), + (namespaces["mathml"], "mo"), + (namespaces["mathml"], "mn"), + (namespaces["mathml"], "ms"), + (namespaces["mathml"], "mtext"), + (namespaces["mathml"], "annotation-xml"), + (namespaces["svg"], "foreignObject"), + (namespaces["svg"], "desc"), + (namespaces["svg"], "title"), +)) + +formattingElements = frozenset(( + (namespaces["html"], "a"), + (namespaces["html"], "b"), + (namespaces["html"], "big"), + (namespaces["html"], "code"), + (namespaces["html"], "em"), + (namespaces["html"], "font"), + (namespaces["html"], "i"), + (namespaces["html"], "nobr"), + (namespaces["html"], "s"), + (namespaces["html"], "small"), + (namespaces["html"], "strike"), + (namespaces["html"], "strong"), + (namespaces["html"], "tt"), + (namespaces["html"], "u") +)) + +specialElements = frozenset(( + (namespaces["html"], "address"), + (namespaces["html"], "applet"), + (namespaces["html"], "area"), + (namespaces["html"], "article"), + (namespaces["html"], "aside"), + (namespaces["html"], "base"), + (namespaces["html"], "basefont"), + (namespaces["html"], "bgsound"), + (namespaces["html"], "blockquote"), + (namespaces["html"], "body"), + (namespaces["html"], "br"), + (namespaces["html"], "button"), + (namespaces["html"], "caption"), + (namespaces["html"], "center"), + (namespaces["html"], "col"), + (namespaces["html"], "colgroup"), + (namespaces["html"], "command"), + (namespaces["html"], "dd"), + (namespaces["html"], "details"), + (namespaces["html"], "dir"), + (namespaces["html"], "div"), + (namespaces["html"], "dl"), + (namespaces["html"], "dt"), + (namespaces["html"], "embed"), + (namespaces["html"], "fieldset"), + (namespaces["html"], "figure"), + (namespaces["html"], "footer"), + (namespaces["html"], "form"), + (namespaces["html"], "frame"), + (namespaces["html"], "frameset"), + (namespaces["html"], "h1"), + (namespaces["html"], "h2"), + (namespaces["html"], "h3"), + (namespaces["html"], "h4"), + (namespaces["html"], "h5"), + (namespaces["html"], "h6"), + (namespaces["html"], "head"), + (namespaces["html"], "header"), + (namespaces["html"], "hr"), + (namespaces["html"], "html"), + (namespaces["html"], "iframe"), + # Note that image is commented out in the spec as "this isn't an + # element that can end up on the stack, so it doesn't matter," + (namespaces["html"], "image"), + (namespaces["html"], "img"), + (namespaces["html"], "input"), + (namespaces["html"], "isindex"), + (namespaces["html"], "li"), + (namespaces["html"], "link"), + (namespaces["html"], "listing"), + (namespaces["html"], "marquee"), + (namespaces["html"], "menu"), + (namespaces["html"], "meta"), + (namespaces["html"], "nav"), + (namespaces["html"], "noembed"), + (namespaces["html"], "noframes"), + (namespaces["html"], "noscript"), + (namespaces["html"], "object"), + (namespaces["html"], "ol"), + (namespaces["html"], "p"), + (namespaces["html"], "param"), + (namespaces["html"], "plaintext"), + (namespaces["html"], "pre"), + (namespaces["html"], "script"), + (namespaces["html"], "section"), + (namespaces["html"], "select"), + (namespaces["html"], "style"), + (namespaces["html"], "table"), + (namespaces["html"], "tbody"), + (namespaces["html"], "td"), + (namespaces["html"], "textarea"), + (namespaces["html"], "tfoot"), + (namespaces["html"], "th"), + (namespaces["html"], "thead"), + (namespaces["html"], "title"), + (namespaces["html"], "tr"), + (namespaces["html"], "ul"), + (namespaces["html"], "wbr"), + (namespaces["html"], "xmp"), + (namespaces["svg"], "foreignObject") +)) + +htmlIntegrationPointElements = frozenset(( + (namespaces["mathml"], "annotaion-xml"), + (namespaces["svg"], "foreignObject"), + (namespaces["svg"], "desc"), + (namespaces["svg"], "title") +)) + +mathmlTextIntegrationPointElements = frozenset(( + (namespaces["mathml"], "mi"), + (namespaces["mathml"], "mo"), + (namespaces["mathml"], "mn"), + (namespaces["mathml"], "ms"), + (namespaces["mathml"], "mtext") +)) + +adjustForeignAttributes = { + "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]), + "xlink:arcrole": ("xlink", "arcrole", namespaces["xlink"]), + "xlink:href": ("xlink", "href", namespaces["xlink"]), + "xlink:role": ("xlink", "role", namespaces["xlink"]), + "xlink:show": ("xlink", "show", namespaces["xlink"]), + "xlink:title": ("xlink", "title", namespaces["xlink"]), + "xlink:type": ("xlink", "type", namespaces["xlink"]), + "xml:base": ("xml", "base", namespaces["xml"]), + "xml:lang": ("xml", "lang", namespaces["xml"]), + "xml:space": ("xml", "space", namespaces["xml"]), + "xmlns": (None, "xmlns", namespaces["xmlns"]), + "xmlns:xlink": ("xmlns", "xlink", namespaces["xmlns"]) +} + +unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in + adjustForeignAttributes.items()]) + +spaceCharacters = frozenset(( + "\t", + "\n", + "\u000C", + " ", + "\r" +)) + +tableInsertModeElements = frozenset(( + "table", + "tbody", + "tfoot", + "thead", + "tr" +)) + +asciiLowercase = frozenset(string.ascii_lowercase) +asciiUppercase = frozenset(string.ascii_uppercase) +asciiLetters = frozenset(string.ascii_letters) +digits = frozenset(string.digits) +hexDigits = frozenset(string.hexdigits) + +asciiUpper2Lower = dict([(ord(c), ord(c.lower())) + for c in string.ascii_uppercase]) + +# Heading elements need to be ordered +headingElements = ( + "h1", + "h2", + "h3", + "h4", + "h5", + "h6" +) + +voidElements = frozenset(( + "base", + "command", + "event-source", + "link", + "meta", + "hr", + "br", + "img", + "embed", + "param", + "area", + "col", + "input", + "source", + "track" +)) + +cdataElements = frozenset(('title', 'textarea')) + +rcdataElements = frozenset(( + 'style', + 'script', + 'xmp', + 'iframe', + 'noembed', + 'noframes', + 'noscript' +)) + +booleanAttributes = { + "": frozenset(("irrelevant",)), + "style": frozenset(("scoped",)), + "img": frozenset(("ismap",)), + "audio": frozenset(("autoplay", "controls")), + "video": frozenset(("autoplay", "controls")), + "script": frozenset(("defer", "async")), + "details": frozenset(("open",)), + "datagrid": frozenset(("multiple", "disabled")), + "command": frozenset(("hidden", "disabled", "checked", "default")), + "hr": frozenset(("noshade")), + "menu": frozenset(("autosubmit",)), + "fieldset": frozenset(("disabled", "readonly")), + "option": frozenset(("disabled", "readonly", "selected")), + "optgroup": frozenset(("disabled", "readonly")), + "button": frozenset(("disabled", "autofocus")), + "input": frozenset(("disabled", "readonly", "required", "autofocus", "checked", "ismap")), + "select": frozenset(("disabled", "readonly", "autofocus", "multiple")), + "output": frozenset(("disabled", "readonly")), +} + +# entitiesWindows1252 has to be _ordered_ and needs to have an index. It +# therefore can't be a frozenset. +entitiesWindows1252 = ( + 8364, # 0x80 0x20AC EURO SIGN + 65533, # 0x81 UNDEFINED + 8218, # 0x82 0x201A SINGLE LOW-9 QUOTATION MARK + 402, # 0x83 0x0192 LATIN SMALL LETTER F WITH HOOK + 8222, # 0x84 0x201E DOUBLE LOW-9 QUOTATION MARK + 8230, # 0x85 0x2026 HORIZONTAL ELLIPSIS + 8224, # 0x86 0x2020 DAGGER + 8225, # 0x87 0x2021 DOUBLE DAGGER + 710, # 0x88 0x02C6 MODIFIER LETTER CIRCUMFLEX ACCENT + 8240, # 0x89 0x2030 PER MILLE SIGN + 352, # 0x8A 0x0160 LATIN CAPITAL LETTER S WITH CARON + 8249, # 0x8B 0x2039 SINGLE LEFT-POINTING ANGLE QUOTATION MARK + 338, # 0x8C 0x0152 LATIN CAPITAL LIGATURE OE + 65533, # 0x8D UNDEFINED + 381, # 0x8E 0x017D LATIN CAPITAL LETTER Z WITH CARON + 65533, # 0x8F UNDEFINED + 65533, # 0x90 UNDEFINED + 8216, # 0x91 0x2018 LEFT SINGLE QUOTATION MARK + 8217, # 0x92 0x2019 RIGHT SINGLE QUOTATION MARK + 8220, # 0x93 0x201C LEFT DOUBLE QUOTATION MARK + 8221, # 0x94 0x201D RIGHT DOUBLE QUOTATION MARK + 8226, # 0x95 0x2022 BULLET + 8211, # 0x96 0x2013 EN DASH + 8212, # 0x97 0x2014 EM DASH + 732, # 0x98 0x02DC SMALL TILDE + 8482, # 0x99 0x2122 TRADE MARK SIGN + 353, # 0x9A 0x0161 LATIN SMALL LETTER S WITH CARON + 8250, # 0x9B 0x203A SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + 339, # 0x9C 0x0153 LATIN SMALL LIGATURE OE + 65533, # 0x9D UNDEFINED + 382, # 0x9E 0x017E LATIN SMALL LETTER Z WITH CARON + 376 # 0x9F 0x0178 LATIN CAPITAL LETTER Y WITH DIAERESIS +) + +xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;')) + +entities = { + "AElig": "\xc6", + "AElig;": "\xc6", + "AMP": "&", + "AMP;": "&", + "Aacute": "\xc1", + "Aacute;": "\xc1", + "Abreve;": "\u0102", + "Acirc": "\xc2", + "Acirc;": "\xc2", + "Acy;": "\u0410", + "Afr;": "\U0001d504", + "Agrave": "\xc0", + "Agrave;": "\xc0", + "Alpha;": "\u0391", + "Amacr;": "\u0100", + "And;": "\u2a53", + "Aogon;": "\u0104", + "Aopf;": "\U0001d538", + "ApplyFunction;": "\u2061", + "Aring": "\xc5", + "Aring;": "\xc5", + "Ascr;": "\U0001d49c", + "Assign;": "\u2254", + "Atilde": "\xc3", + "Atilde;": "\xc3", + "Auml": "\xc4", + "Auml;": "\xc4", + "Backslash;": "\u2216", + "Barv;": "\u2ae7", + "Barwed;": "\u2306", + "Bcy;": "\u0411", + "Because;": "\u2235", + "Bernoullis;": "\u212c", + "Beta;": "\u0392", + "Bfr;": "\U0001d505", + "Bopf;": "\U0001d539", + "Breve;": "\u02d8", + "Bscr;": "\u212c", + "Bumpeq;": "\u224e", + "CHcy;": "\u0427", + "COPY": "\xa9", + "COPY;": "\xa9", + "Cacute;": "\u0106", + "Cap;": "\u22d2", + "CapitalDifferentialD;": "\u2145", + "Cayleys;": "\u212d", + "Ccaron;": "\u010c", + "Ccedil": "\xc7", + "Ccedil;": "\xc7", + "Ccirc;": "\u0108", + "Cconint;": "\u2230", + "Cdot;": "\u010a", + "Cedilla;": "\xb8", + "CenterDot;": "\xb7", + "Cfr;": "\u212d", + "Chi;": "\u03a7", + "CircleDot;": "\u2299", + "CircleMinus;": "\u2296", + "CirclePlus;": "\u2295", + "CircleTimes;": "\u2297", + "ClockwiseContourIntegral;": "\u2232", + "CloseCurlyDoubleQuote;": "\u201d", + "CloseCurlyQuote;": "\u2019", + "Colon;": "\u2237", + "Colone;": "\u2a74", + "Congruent;": "\u2261", + "Conint;": "\u222f", + "ContourIntegral;": "\u222e", + "Copf;": "\u2102", + "Coproduct;": "\u2210", + "CounterClockwiseContourIntegral;": "\u2233", + "Cross;": "\u2a2f", + "Cscr;": "\U0001d49e", + "Cup;": "\u22d3", + "CupCap;": "\u224d", + "DD;": "\u2145", + "DDotrahd;": "\u2911", + "DJcy;": "\u0402", + "DScy;": "\u0405", + "DZcy;": "\u040f", + "Dagger;": "\u2021", + "Darr;": "\u21a1", + "Dashv;": "\u2ae4", + "Dcaron;": "\u010e", + "Dcy;": "\u0414", + "Del;": "\u2207", + "Delta;": "\u0394", + "Dfr;": "\U0001d507", + "DiacriticalAcute;": "\xb4", + "DiacriticalDot;": "\u02d9", + "DiacriticalDoubleAcute;": "\u02dd", + "DiacriticalGrave;": "`", + "DiacriticalTilde;": "\u02dc", + "Diamond;": "\u22c4", + "DifferentialD;": "\u2146", + "Dopf;": "\U0001d53b", + "Dot;": "\xa8", + "DotDot;": "\u20dc", + "DotEqual;": "\u2250", + "DoubleContourIntegral;": "\u222f", + "DoubleDot;": "\xa8", + "DoubleDownArrow;": "\u21d3", + "DoubleLeftArrow;": "\u21d0", + "DoubleLeftRightArrow;": "\u21d4", + "DoubleLeftTee;": "\u2ae4", + "DoubleLongLeftArrow;": "\u27f8", + "DoubleLongLeftRightArrow;": "\u27fa", + "DoubleLongRightArrow;": "\u27f9", + "DoubleRightArrow;": "\u21d2", + "DoubleRightTee;": "\u22a8", + "DoubleUpArrow;": "\u21d1", + "DoubleUpDownArrow;": "\u21d5", + "DoubleVerticalBar;": "\u2225", + "DownArrow;": "\u2193", + "DownArrowBar;": "\u2913", + "DownArrowUpArrow;": "\u21f5", + "DownBreve;": "\u0311", + "DownLeftRightVector;": "\u2950", + "DownLeftTeeVector;": "\u295e", + "DownLeftVector;": "\u21bd", + "DownLeftVectorBar;": "\u2956", + "DownRightTeeVector;": "\u295f", + "DownRightVector;": "\u21c1", + "DownRightVectorBar;": "\u2957", + "DownTee;": "\u22a4", + "DownTeeArrow;": "\u21a7", + "Downarrow;": "\u21d3", + "Dscr;": "\U0001d49f", + "Dstrok;": "\u0110", + "ENG;": "\u014a", + "ETH": "\xd0", + "ETH;": "\xd0", + "Eacute": "\xc9", + "Eacute;": "\xc9", + "Ecaron;": "\u011a", + "Ecirc": "\xca", + "Ecirc;": "\xca", + "Ecy;": "\u042d", + "Edot;": "\u0116", + "Efr;": "\U0001d508", + "Egrave": "\xc8", + "Egrave;": "\xc8", + "Element;": "\u2208", + "Emacr;": "\u0112", + "EmptySmallSquare;": "\u25fb", + "EmptyVerySmallSquare;": "\u25ab", + "Eogon;": "\u0118", + "Eopf;": "\U0001d53c", + "Epsilon;": "\u0395", + "Equal;": "\u2a75", + "EqualTilde;": "\u2242", + "Equilibrium;": "\u21cc", + "Escr;": "\u2130", + "Esim;": "\u2a73", + "Eta;": "\u0397", + "Euml": "\xcb", + "Euml;": "\xcb", + "Exists;": "\u2203", + "ExponentialE;": "\u2147", + "Fcy;": "\u0424", + "Ffr;": "\U0001d509", + "FilledSmallSquare;": "\u25fc", + "FilledVerySmallSquare;": "\u25aa", + "Fopf;": "\U0001d53d", + "ForAll;": "\u2200", + "Fouriertrf;": "\u2131", + "Fscr;": "\u2131", + "GJcy;": "\u0403", + "GT": ">", + "GT;": ">", + "Gamma;": "\u0393", + "Gammad;": "\u03dc", + "Gbreve;": "\u011e", + "Gcedil;": "\u0122", + "Gcirc;": "\u011c", + "Gcy;": "\u0413", + "Gdot;": "\u0120", + "Gfr;": "\U0001d50a", + "Gg;": "\u22d9", + "Gopf;": "\U0001d53e", + "GreaterEqual;": "\u2265", + "GreaterEqualLess;": "\u22db", + "GreaterFullEqual;": "\u2267", + "GreaterGreater;": "\u2aa2", + "GreaterLess;": "\u2277", + "GreaterSlantEqual;": "\u2a7e", + "GreaterTilde;": "\u2273", + "Gscr;": "\U0001d4a2", + "Gt;": "\u226b", + "HARDcy;": "\u042a", + "Hacek;": "\u02c7", + "Hat;": "^", + "Hcirc;": "\u0124", + "Hfr;": "\u210c", + "HilbertSpace;": "\u210b", + "Hopf;": "\u210d", + "HorizontalLine;": "\u2500", + "Hscr;": "\u210b", + "Hstrok;": "\u0126", + "HumpDownHump;": "\u224e", + "HumpEqual;": "\u224f", + "IEcy;": "\u0415", + "IJlig;": "\u0132", + "IOcy;": "\u0401", + "Iacute": "\xcd", + "Iacute;": "\xcd", + "Icirc": "\xce", + "Icirc;": "\xce", + "Icy;": "\u0418", + "Idot;": "\u0130", + "Ifr;": "\u2111", + "Igrave": "\xcc", + "Igrave;": "\xcc", + "Im;": "\u2111", + "Imacr;": "\u012a", + "ImaginaryI;": "\u2148", + "Implies;": "\u21d2", + "Int;": "\u222c", + "Integral;": "\u222b", + "Intersection;": "\u22c2", + "InvisibleComma;": "\u2063", + "InvisibleTimes;": "\u2062", + "Iogon;": "\u012e", + "Iopf;": "\U0001d540", + "Iota;": "\u0399", + "Iscr;": "\u2110", + "Itilde;": "\u0128", + "Iukcy;": "\u0406", + "Iuml": "\xcf", + "Iuml;": "\xcf", + "Jcirc;": "\u0134", + "Jcy;": "\u0419", + "Jfr;": "\U0001d50d", + "Jopf;": "\U0001d541", + "Jscr;": "\U0001d4a5", + "Jsercy;": "\u0408", + "Jukcy;": "\u0404", + "KHcy;": "\u0425", + "KJcy;": "\u040c", + "Kappa;": "\u039a", + "Kcedil;": "\u0136", + "Kcy;": "\u041a", + "Kfr;": "\U0001d50e", + "Kopf;": "\U0001d542", + "Kscr;": "\U0001d4a6", + "LJcy;": "\u0409", + "LT": "<", + "LT;": "<", + "Lacute;": "\u0139", + "Lambda;": "\u039b", + "Lang;": "\u27ea", + "Laplacetrf;": "\u2112", + "Larr;": "\u219e", + "Lcaron;": "\u013d", + "Lcedil;": "\u013b", + "Lcy;": "\u041b", + "LeftAngleBracket;": "\u27e8", + "LeftArrow;": "\u2190", + "LeftArrowBar;": "\u21e4", + "LeftArrowRightArrow;": "\u21c6", + "LeftCeiling;": "\u2308", + "LeftDoubleBracket;": "\u27e6", + "LeftDownTeeVector;": "\u2961", + "LeftDownVector;": "\u21c3", + "LeftDownVectorBar;": "\u2959", + "LeftFloor;": "\u230a", + "LeftRightArrow;": "\u2194", + "LeftRightVector;": "\u294e", + "LeftTee;": "\u22a3", + "LeftTeeArrow;": "\u21a4", + "LeftTeeVector;": "\u295a", + "LeftTriangle;": "\u22b2", + "LeftTriangleBar;": "\u29cf", + "LeftTriangleEqual;": "\u22b4", + "LeftUpDownVector;": "\u2951", + "LeftUpTeeVector;": "\u2960", + "LeftUpVector;": "\u21bf", + "LeftUpVectorBar;": "\u2958", + "LeftVector;": "\u21bc", + "LeftVectorBar;": "\u2952", + "Leftarrow;": "\u21d0", + "Leftrightarrow;": "\u21d4", + "LessEqualGreater;": "\u22da", + "LessFullEqual;": "\u2266", + "LessGreater;": "\u2276", + "LessLess;": "\u2aa1", + "LessSlantEqual;": "\u2a7d", + "LessTilde;": "\u2272", + "Lfr;": "\U0001d50f", + "Ll;": "\u22d8", + "Lleftarrow;": "\u21da", + "Lmidot;": "\u013f", + "LongLeftArrow;": "\u27f5", + "LongLeftRightArrow;": "\u27f7", + "LongRightArrow;": "\u27f6", + "Longleftarrow;": "\u27f8", + "Longleftrightarrow;": "\u27fa", + "Longrightarrow;": "\u27f9", + "Lopf;": "\U0001d543", + "LowerLeftArrow;": "\u2199", + "LowerRightArrow;": "\u2198", + "Lscr;": "\u2112", + "Lsh;": "\u21b0", + "Lstrok;": "\u0141", + "Lt;": "\u226a", + "Map;": "\u2905", + "Mcy;": "\u041c", + "MediumSpace;": "\u205f", + "Mellintrf;": "\u2133", + "Mfr;": "\U0001d510", + "MinusPlus;": "\u2213", + "Mopf;": "\U0001d544", + "Mscr;": "\u2133", + "Mu;": "\u039c", + "NJcy;": "\u040a", + "Nacute;": "\u0143", + "Ncaron;": "\u0147", + "Ncedil;": "\u0145", + "Ncy;": "\u041d", + "NegativeMediumSpace;": "\u200b", + "NegativeThickSpace;": "\u200b", + "NegativeThinSpace;": "\u200b", + "NegativeVeryThinSpace;": "\u200b", + "NestedGreaterGreater;": "\u226b", + "NestedLessLess;": "\u226a", + "NewLine;": "\n", + "Nfr;": "\U0001d511", + "NoBreak;": "\u2060", + "NonBreakingSpace;": "\xa0", + "Nopf;": "\u2115", + "Not;": "\u2aec", + "NotCongruent;": "\u2262", + "NotCupCap;": "\u226d", + "NotDoubleVerticalBar;": "\u2226", + "NotElement;": "\u2209", + "NotEqual;": "\u2260", + "NotEqualTilde;": "\u2242\u0338", + "NotExists;": "\u2204", + "NotGreater;": "\u226f", + "NotGreaterEqual;": "\u2271", + "NotGreaterFullEqual;": "\u2267\u0338", + "NotGreaterGreater;": "\u226b\u0338", + "NotGreaterLess;": "\u2279", + "NotGreaterSlantEqual;": "\u2a7e\u0338", + "NotGreaterTilde;": "\u2275", + "NotHumpDownHump;": "\u224e\u0338", + "NotHumpEqual;": "\u224f\u0338", + "NotLeftTriangle;": "\u22ea", + "NotLeftTriangleBar;": "\u29cf\u0338", + "NotLeftTriangleEqual;": "\u22ec", + "NotLess;": "\u226e", + "NotLessEqual;": "\u2270", + "NotLessGreater;": "\u2278", + "NotLessLess;": "\u226a\u0338", + "NotLessSlantEqual;": "\u2a7d\u0338", + "NotLessTilde;": "\u2274", + "NotNestedGreaterGreater;": "\u2aa2\u0338", + "NotNestedLessLess;": "\u2aa1\u0338", + "NotPrecedes;": "\u2280", + "NotPrecedesEqual;": "\u2aaf\u0338", + "NotPrecedesSlantEqual;": "\u22e0", + "NotReverseElement;": "\u220c", + "NotRightTriangle;": "\u22eb", + "NotRightTriangleBar;": "\u29d0\u0338", + "NotRightTriangleEqual;": "\u22ed", + "NotSquareSubset;": "\u228f\u0338", + "NotSquareSubsetEqual;": "\u22e2", + "NotSquareSuperset;": "\u2290\u0338", + "NotSquareSupersetEqual;": "\u22e3", + "NotSubset;": "\u2282\u20d2", + "NotSubsetEqual;": "\u2288", + "NotSucceeds;": "\u2281", + "NotSucceedsEqual;": "\u2ab0\u0338", + "NotSucceedsSlantEqual;": "\u22e1", + "NotSucceedsTilde;": "\u227f\u0338", + "NotSuperset;": "\u2283\u20d2", + "NotSupersetEqual;": "\u2289", + "NotTilde;": "\u2241", + "NotTildeEqual;": "\u2244", + "NotTildeFullEqual;": "\u2247", + "NotTildeTilde;": "\u2249", + "NotVerticalBar;": "\u2224", + "Nscr;": "\U0001d4a9", + "Ntilde": "\xd1", + "Ntilde;": "\xd1", + "Nu;": "\u039d", + "OElig;": "\u0152", + "Oacute": "\xd3", + "Oacute;": "\xd3", + "Ocirc": "\xd4", + "Ocirc;": "\xd4", + "Ocy;": "\u041e", + "Odblac;": "\u0150", + "Ofr;": "\U0001d512", + "Ograve": "\xd2", + "Ograve;": "\xd2", + "Omacr;": "\u014c", + "Omega;": "\u03a9", + "Omicron;": "\u039f", + "Oopf;": "\U0001d546", + "OpenCurlyDoubleQuote;": "\u201c", + "OpenCurlyQuote;": "\u2018", + "Or;": "\u2a54", + "Oscr;": "\U0001d4aa", + "Oslash": "\xd8", + "Oslash;": "\xd8", + "Otilde": "\xd5", + "Otilde;": "\xd5", + "Otimes;": "\u2a37", + "Ouml": "\xd6", + "Ouml;": "\xd6", + "OverBar;": "\u203e", + "OverBrace;": "\u23de", + "OverBracket;": "\u23b4", + "OverParenthesis;": "\u23dc", + "PartialD;": "\u2202", + "Pcy;": "\u041f", + "Pfr;": "\U0001d513", + "Phi;": "\u03a6", + "Pi;": "\u03a0", + "PlusMinus;": "\xb1", + "Poincareplane;": "\u210c", + "Popf;": "\u2119", + "Pr;": "\u2abb", + "Precedes;": "\u227a", + "PrecedesEqual;": "\u2aaf", + "PrecedesSlantEqual;": "\u227c", + "PrecedesTilde;": "\u227e", + "Prime;": "\u2033", + "Product;": "\u220f", + "Proportion;": "\u2237", + "Proportional;": "\u221d", + "Pscr;": "\U0001d4ab", + "Psi;": "\u03a8", + "QUOT": "\"", + "QUOT;": "\"", + "Qfr;": "\U0001d514", + "Qopf;": "\u211a", + "Qscr;": "\U0001d4ac", + "RBarr;": "\u2910", + "REG": "\xae", + "REG;": "\xae", + "Racute;": "\u0154", + "Rang;": "\u27eb", + "Rarr;": "\u21a0", + "Rarrtl;": "\u2916", + "Rcaron;": "\u0158", + "Rcedil;": "\u0156", + "Rcy;": "\u0420", + "Re;": "\u211c", + "ReverseElement;": "\u220b", + "ReverseEquilibrium;": "\u21cb", + "ReverseUpEquilibrium;": "\u296f", + "Rfr;": "\u211c", + "Rho;": "\u03a1", + "RightAngleBracket;": "\u27e9", + "RightArrow;": "\u2192", + "RightArrowBar;": "\u21e5", + "RightArrowLeftArrow;": "\u21c4", + "RightCeiling;": "\u2309", + "RightDoubleBracket;": "\u27e7", + "RightDownTeeVector;": "\u295d", + "RightDownVector;": "\u21c2", + "RightDownVectorBar;": "\u2955", + "RightFloor;": "\u230b", + "RightTee;": "\u22a2", + "RightTeeArrow;": "\u21a6", + "RightTeeVector;": "\u295b", + "RightTriangle;": "\u22b3", + "RightTriangleBar;": "\u29d0", + "RightTriangleEqual;": "\u22b5", + "RightUpDownVector;": "\u294f", + "RightUpTeeVector;": "\u295c", + "RightUpVector;": "\u21be", + "RightUpVectorBar;": "\u2954", + "RightVector;": "\u21c0", + "RightVectorBar;": "\u2953", + "Rightarrow;": "\u21d2", + "Ropf;": "\u211d", + "RoundImplies;": "\u2970", + "Rrightarrow;": "\u21db", + "Rscr;": "\u211b", + "Rsh;": "\u21b1", + "RuleDelayed;": "\u29f4", + "SHCHcy;": "\u0429", + "SHcy;": "\u0428", + "SOFTcy;": "\u042c", + "Sacute;": "\u015a", + "Sc;": "\u2abc", + "Scaron;": "\u0160", + "Scedil;": "\u015e", + "Scirc;": "\u015c", + "Scy;": "\u0421", + "Sfr;": "\U0001d516", + "ShortDownArrow;": "\u2193", + "ShortLeftArrow;": "\u2190", + "ShortRightArrow;": "\u2192", + "ShortUpArrow;": "\u2191", + "Sigma;": "\u03a3", + "SmallCircle;": "\u2218", + "Sopf;": "\U0001d54a", + "Sqrt;": "\u221a", + "Square;": "\u25a1", + "SquareIntersection;": "\u2293", + "SquareSubset;": "\u228f", + "SquareSubsetEqual;": "\u2291", + "SquareSuperset;": "\u2290", + "SquareSupersetEqual;": "\u2292", + "SquareUnion;": "\u2294", + "Sscr;": "\U0001d4ae", + "Star;": "\u22c6", + "Sub;": "\u22d0", + "Subset;": "\u22d0", + "SubsetEqual;": "\u2286", + "Succeeds;": "\u227b", + "SucceedsEqual;": "\u2ab0", + "SucceedsSlantEqual;": "\u227d", + "SucceedsTilde;": "\u227f", + "SuchThat;": "\u220b", + "Sum;": "\u2211", + "Sup;": "\u22d1", + "Superset;": "\u2283", + "SupersetEqual;": "\u2287", + "Supset;": "\u22d1", + "THORN": "\xde", + "THORN;": "\xde", + "TRADE;": "\u2122", + "TSHcy;": "\u040b", + "TScy;": "\u0426", + "Tab;": "\t", + "Tau;": "\u03a4", + "Tcaron;": "\u0164", + "Tcedil;": "\u0162", + "Tcy;": "\u0422", + "Tfr;": "\U0001d517", + "Therefore;": "\u2234", + "Theta;": "\u0398", + "ThickSpace;": "\u205f\u200a", + "ThinSpace;": "\u2009", + "Tilde;": "\u223c", + "TildeEqual;": "\u2243", + "TildeFullEqual;": "\u2245", + "TildeTilde;": "\u2248", + "Topf;": "\U0001d54b", + "TripleDot;": "\u20db", + "Tscr;": "\U0001d4af", + "Tstrok;": "\u0166", + "Uacute": "\xda", + "Uacute;": "\xda", + "Uarr;": "\u219f", + "Uarrocir;": "\u2949", + "Ubrcy;": "\u040e", + "Ubreve;": "\u016c", + "Ucirc": "\xdb", + "Ucirc;": "\xdb", + "Ucy;": "\u0423", + "Udblac;": "\u0170", + "Ufr;": "\U0001d518", + "Ugrave": "\xd9", + "Ugrave;": "\xd9", + "Umacr;": "\u016a", + "UnderBar;": "_", + "UnderBrace;": "\u23df", + "UnderBracket;": "\u23b5", + "UnderParenthesis;": "\u23dd", + "Union;": "\u22c3", + "UnionPlus;": "\u228e", + "Uogon;": "\u0172", + "Uopf;": "\U0001d54c", + "UpArrow;": "\u2191", + "UpArrowBar;": "\u2912", + "UpArrowDownArrow;": "\u21c5", + "UpDownArrow;": "\u2195", + "UpEquilibrium;": "\u296e", + "UpTee;": "\u22a5", + "UpTeeArrow;": "\u21a5", + "Uparrow;": "\u21d1", + "Updownarrow;": "\u21d5", + "UpperLeftArrow;": "\u2196", + "UpperRightArrow;": "\u2197", + "Upsi;": "\u03d2", + "Upsilon;": "\u03a5", + "Uring;": "\u016e", + "Uscr;": "\U0001d4b0", + "Utilde;": "\u0168", + "Uuml": "\xdc", + "Uuml;": "\xdc", + "VDash;": "\u22ab", + "Vbar;": "\u2aeb", + "Vcy;": "\u0412", + "Vdash;": "\u22a9", + "Vdashl;": "\u2ae6", + "Vee;": "\u22c1", + "Verbar;": "\u2016", + "Vert;": "\u2016", + "VerticalBar;": "\u2223", + "VerticalLine;": "|", + "VerticalSeparator;": "\u2758", + "VerticalTilde;": "\u2240", + "VeryThinSpace;": "\u200a", + "Vfr;": "\U0001d519", + "Vopf;": "\U0001d54d", + "Vscr;": "\U0001d4b1", + "Vvdash;": "\u22aa", + "Wcirc;": "\u0174", + "Wedge;": "\u22c0", + "Wfr;": "\U0001d51a", + "Wopf;": "\U0001d54e", + "Wscr;": "\U0001d4b2", + "Xfr;": "\U0001d51b", + "Xi;": "\u039e", + "Xopf;": "\U0001d54f", + "Xscr;": "\U0001d4b3", + "YAcy;": "\u042f", + "YIcy;": "\u0407", + "YUcy;": "\u042e", + "Yacute": "\xdd", + "Yacute;": "\xdd", + "Ycirc;": "\u0176", + "Ycy;": "\u042b", + "Yfr;": "\U0001d51c", + "Yopf;": "\U0001d550", + "Yscr;": "\U0001d4b4", + "Yuml;": "\u0178", + "ZHcy;": "\u0416", + "Zacute;": "\u0179", + "Zcaron;": "\u017d", + "Zcy;": "\u0417", + "Zdot;": "\u017b", + "ZeroWidthSpace;": "\u200b", + "Zeta;": "\u0396", + "Zfr;": "\u2128", + "Zopf;": "\u2124", + "Zscr;": "\U0001d4b5", + "aacute": "\xe1", + "aacute;": "\xe1", + "abreve;": "\u0103", + "ac;": "\u223e", + "acE;": "\u223e\u0333", + "acd;": "\u223f", + "acirc": "\xe2", + "acirc;": "\xe2", + "acute": "\xb4", + "acute;": "\xb4", + "acy;": "\u0430", + "aelig": "\xe6", + "aelig;": "\xe6", + "af;": "\u2061", + "afr;": "\U0001d51e", + "agrave": "\xe0", + "agrave;": "\xe0", + "alefsym;": "\u2135", + "aleph;": "\u2135", + "alpha;": "\u03b1", + "amacr;": "\u0101", + "amalg;": "\u2a3f", + "amp": "&", + "amp;": "&", + "and;": "\u2227", + "andand;": "\u2a55", + "andd;": "\u2a5c", + "andslope;": "\u2a58", + "andv;": "\u2a5a", + "ang;": "\u2220", + "ange;": "\u29a4", + "angle;": "\u2220", + "angmsd;": "\u2221", + "angmsdaa;": "\u29a8", + "angmsdab;": "\u29a9", + "angmsdac;": "\u29aa", + "angmsdad;": "\u29ab", + "angmsdae;": "\u29ac", + "angmsdaf;": "\u29ad", + "angmsdag;": "\u29ae", + "angmsdah;": "\u29af", + "angrt;": "\u221f", + "angrtvb;": "\u22be", + "angrtvbd;": "\u299d", + "angsph;": "\u2222", + "angst;": "\xc5", + "angzarr;": "\u237c", + "aogon;": "\u0105", + "aopf;": "\U0001d552", + "ap;": "\u2248", + "apE;": "\u2a70", + "apacir;": "\u2a6f", + "ape;": "\u224a", + "apid;": "\u224b", + "apos;": "'", + "approx;": "\u2248", + "approxeq;": "\u224a", + "aring": "\xe5", + "aring;": "\xe5", + "ascr;": "\U0001d4b6", + "ast;": "*", + "asymp;": "\u2248", + "asympeq;": "\u224d", + "atilde": "\xe3", + "atilde;": "\xe3", + "auml": "\xe4", + "auml;": "\xe4", + "awconint;": "\u2233", + "awint;": "\u2a11", + "bNot;": "\u2aed", + "backcong;": "\u224c", + "backepsilon;": "\u03f6", + "backprime;": "\u2035", + "backsim;": "\u223d", + "backsimeq;": "\u22cd", + "barvee;": "\u22bd", + "barwed;": "\u2305", + "barwedge;": "\u2305", + "bbrk;": "\u23b5", + "bbrktbrk;": "\u23b6", + "bcong;": "\u224c", + "bcy;": "\u0431", + "bdquo;": "\u201e", + "becaus;": "\u2235", + "because;": "\u2235", + "bemptyv;": "\u29b0", + "bepsi;": "\u03f6", + "bernou;": "\u212c", + "beta;": "\u03b2", + "beth;": "\u2136", + "between;": "\u226c", + "bfr;": "\U0001d51f", + "bigcap;": "\u22c2", + "bigcirc;": "\u25ef", + "bigcup;": "\u22c3", + "bigodot;": "\u2a00", + "bigoplus;": "\u2a01", + "bigotimes;": "\u2a02", + "bigsqcup;": "\u2a06", + "bigstar;": "\u2605", + "bigtriangledown;": "\u25bd", + "bigtriangleup;": "\u25b3", + "biguplus;": "\u2a04", + "bigvee;": "\u22c1", + "bigwedge;": "\u22c0", + "bkarow;": "\u290d", + "blacklozenge;": "\u29eb", + "blacksquare;": "\u25aa", + "blacktriangle;": "\u25b4", + "blacktriangledown;": "\u25be", + "blacktriangleleft;": "\u25c2", + "blacktriangleright;": "\u25b8", + "blank;": "\u2423", + "blk12;": "\u2592", + "blk14;": "\u2591", + "blk34;": "\u2593", + "block;": "\u2588", + "bne;": "=\u20e5", + "bnequiv;": "\u2261\u20e5", + "bnot;": "\u2310", + "bopf;": "\U0001d553", + "bot;": "\u22a5", + "bottom;": "\u22a5", + "bowtie;": "\u22c8", + "boxDL;": "\u2557", + "boxDR;": "\u2554", + "boxDl;": "\u2556", + "boxDr;": "\u2553", + "boxH;": "\u2550", + "boxHD;": "\u2566", + "boxHU;": "\u2569", + "boxHd;": "\u2564", + "boxHu;": "\u2567", + "boxUL;": "\u255d", + "boxUR;": "\u255a", + "boxUl;": "\u255c", + "boxUr;": "\u2559", + "boxV;": "\u2551", + "boxVH;": "\u256c", + "boxVL;": "\u2563", + "boxVR;": "\u2560", + "boxVh;": "\u256b", + "boxVl;": "\u2562", + "boxVr;": "\u255f", + "boxbox;": "\u29c9", + "boxdL;": "\u2555", + "boxdR;": "\u2552", + "boxdl;": "\u2510", + "boxdr;": "\u250c", + "boxh;": "\u2500", + "boxhD;": "\u2565", + "boxhU;": "\u2568", + "boxhd;": "\u252c", + "boxhu;": "\u2534", + "boxminus;": "\u229f", + "boxplus;": "\u229e", + "boxtimes;": "\u22a0", + "boxuL;": "\u255b", + "boxuR;": "\u2558", + "boxul;": "\u2518", + "boxur;": "\u2514", + "boxv;": "\u2502", + "boxvH;": "\u256a", + "boxvL;": "\u2561", + "boxvR;": "\u255e", + "boxvh;": "\u253c", + "boxvl;": "\u2524", + "boxvr;": "\u251c", + "bprime;": "\u2035", + "breve;": "\u02d8", + "brvbar": "\xa6", + "brvbar;": "\xa6", + "bscr;": "\U0001d4b7", + "bsemi;": "\u204f", + "bsim;": "\u223d", + "bsime;": "\u22cd", + "bsol;": "\\", + "bsolb;": "\u29c5", + "bsolhsub;": "\u27c8", + "bull;": "\u2022", + "bullet;": "\u2022", + "bump;": "\u224e", + "bumpE;": "\u2aae", + "bumpe;": "\u224f", + "bumpeq;": "\u224f", + "cacute;": "\u0107", + "cap;": "\u2229", + "capand;": "\u2a44", + "capbrcup;": "\u2a49", + "capcap;": "\u2a4b", + "capcup;": "\u2a47", + "capdot;": "\u2a40", + "caps;": "\u2229\ufe00", + "caret;": "\u2041", + "caron;": "\u02c7", + "ccaps;": "\u2a4d", + "ccaron;": "\u010d", + "ccedil": "\xe7", + "ccedil;": "\xe7", + "ccirc;": "\u0109", + "ccups;": "\u2a4c", + "ccupssm;": "\u2a50", + "cdot;": "\u010b", + "cedil": "\xb8", + "cedil;": "\xb8", + "cemptyv;": "\u29b2", + "cent": "\xa2", + "cent;": "\xa2", + "centerdot;": "\xb7", + "cfr;": "\U0001d520", + "chcy;": "\u0447", + "check;": "\u2713", + "checkmark;": "\u2713", + "chi;": "\u03c7", + "cir;": "\u25cb", + "cirE;": "\u29c3", + "circ;": "\u02c6", + "circeq;": "\u2257", + "circlearrowleft;": "\u21ba", + "circlearrowright;": "\u21bb", + "circledR;": "\xae", + "circledS;": "\u24c8", + "circledast;": "\u229b", + "circledcirc;": "\u229a", + "circleddash;": "\u229d", + "cire;": "\u2257", + "cirfnint;": "\u2a10", + "cirmid;": "\u2aef", + "cirscir;": "\u29c2", + "clubs;": "\u2663", + "clubsuit;": "\u2663", + "colon;": ":", + "colone;": "\u2254", + "coloneq;": "\u2254", + "comma;": ",", + "commat;": "@", + "comp;": "\u2201", + "compfn;": "\u2218", + "complement;": "\u2201", + "complexes;": "\u2102", + "cong;": "\u2245", + "congdot;": "\u2a6d", + "conint;": "\u222e", + "copf;": "\U0001d554", + "coprod;": "\u2210", + "copy": "\xa9", + "copy;": "\xa9", + "copysr;": "\u2117", + "crarr;": "\u21b5", + "cross;": "\u2717", + "cscr;": "\U0001d4b8", + "csub;": "\u2acf", + "csube;": "\u2ad1", + "csup;": "\u2ad0", + "csupe;": "\u2ad2", + "ctdot;": "\u22ef", + "cudarrl;": "\u2938", + "cudarrr;": "\u2935", + "cuepr;": "\u22de", + "cuesc;": "\u22df", + "cularr;": "\u21b6", + "cularrp;": "\u293d", + "cup;": "\u222a", + "cupbrcap;": "\u2a48", + "cupcap;": "\u2a46", + "cupcup;": "\u2a4a", + "cupdot;": "\u228d", + "cupor;": "\u2a45", + "cups;": "\u222a\ufe00", + "curarr;": "\u21b7", + "curarrm;": "\u293c", + "curlyeqprec;": "\u22de", + "curlyeqsucc;": "\u22df", + "curlyvee;": "\u22ce", + "curlywedge;": "\u22cf", + "curren": "\xa4", + "curren;": "\xa4", + "curvearrowleft;": "\u21b6", + "curvearrowright;": "\u21b7", + "cuvee;": "\u22ce", + "cuwed;": "\u22cf", + "cwconint;": "\u2232", + "cwint;": "\u2231", + "cylcty;": "\u232d", + "dArr;": "\u21d3", + "dHar;": "\u2965", + "dagger;": "\u2020", + "daleth;": "\u2138", + "darr;": "\u2193", + "dash;": "\u2010", + "dashv;": "\u22a3", + "dbkarow;": "\u290f", + "dblac;": "\u02dd", + "dcaron;": "\u010f", + "dcy;": "\u0434", + "dd;": "\u2146", + "ddagger;": "\u2021", + "ddarr;": "\u21ca", + "ddotseq;": "\u2a77", + "deg": "\xb0", + "deg;": "\xb0", + "delta;": "\u03b4", + "demptyv;": "\u29b1", + "dfisht;": "\u297f", + "dfr;": "\U0001d521", + "dharl;": "\u21c3", + "dharr;": "\u21c2", + "diam;": "\u22c4", + "diamond;": "\u22c4", + "diamondsuit;": "\u2666", + "diams;": "\u2666", + "die;": "\xa8", + "digamma;": "\u03dd", + "disin;": "\u22f2", + "div;": "\xf7", + "divide": "\xf7", + "divide;": "\xf7", + "divideontimes;": "\u22c7", + "divonx;": "\u22c7", + "djcy;": "\u0452", + "dlcorn;": "\u231e", + "dlcrop;": "\u230d", + "dollar;": "$", + "dopf;": "\U0001d555", + "dot;": "\u02d9", + "doteq;": "\u2250", + "doteqdot;": "\u2251", + "dotminus;": "\u2238", + "dotplus;": "\u2214", + "dotsquare;": "\u22a1", + "doublebarwedge;": "\u2306", + "downarrow;": "\u2193", + "downdownarrows;": "\u21ca", + "downharpoonleft;": "\u21c3", + "downharpoonright;": "\u21c2", + "drbkarow;": "\u2910", + "drcorn;": "\u231f", + "drcrop;": "\u230c", + "dscr;": "\U0001d4b9", + "dscy;": "\u0455", + "dsol;": "\u29f6", + "dstrok;": "\u0111", + "dtdot;": "\u22f1", + "dtri;": "\u25bf", + "dtrif;": "\u25be", + "duarr;": "\u21f5", + "duhar;": "\u296f", + "dwangle;": "\u29a6", + "dzcy;": "\u045f", + "dzigrarr;": "\u27ff", + "eDDot;": "\u2a77", + "eDot;": "\u2251", + "eacute": "\xe9", + "eacute;": "\xe9", + "easter;": "\u2a6e", + "ecaron;": "\u011b", + "ecir;": "\u2256", + "ecirc": "\xea", + "ecirc;": "\xea", + "ecolon;": "\u2255", + "ecy;": "\u044d", + "edot;": "\u0117", + "ee;": "\u2147", + "efDot;": "\u2252", + "efr;": "\U0001d522", + "eg;": "\u2a9a", + "egrave": "\xe8", + "egrave;": "\xe8", + "egs;": "\u2a96", + "egsdot;": "\u2a98", + "el;": "\u2a99", + "elinters;": "\u23e7", + "ell;": "\u2113", + "els;": "\u2a95", + "elsdot;": "\u2a97", + "emacr;": "\u0113", + "empty;": "\u2205", + "emptyset;": "\u2205", + "emptyv;": "\u2205", + "emsp13;": "\u2004", + "emsp14;": "\u2005", + "emsp;": "\u2003", + "eng;": "\u014b", + "ensp;": "\u2002", + "eogon;": "\u0119", + "eopf;": "\U0001d556", + "epar;": "\u22d5", + "eparsl;": "\u29e3", + "eplus;": "\u2a71", + "epsi;": "\u03b5", + "epsilon;": "\u03b5", + "epsiv;": "\u03f5", + "eqcirc;": "\u2256", + "eqcolon;": "\u2255", + "eqsim;": "\u2242", + "eqslantgtr;": "\u2a96", + "eqslantless;": "\u2a95", + "equals;": "=", + "equest;": "\u225f", + "equiv;": "\u2261", + "equivDD;": "\u2a78", + "eqvparsl;": "\u29e5", + "erDot;": "\u2253", + "erarr;": "\u2971", + "escr;": "\u212f", + "esdot;": "\u2250", + "esim;": "\u2242", + "eta;": "\u03b7", + "eth": "\xf0", + "eth;": "\xf0", + "euml": "\xeb", + "euml;": "\xeb", + "euro;": "\u20ac", + "excl;": "!", + "exist;": "\u2203", + "expectation;": "\u2130", + "exponentiale;": "\u2147", + "fallingdotseq;": "\u2252", + "fcy;": "\u0444", + "female;": "\u2640", + "ffilig;": "\ufb03", + "fflig;": "\ufb00", + "ffllig;": "\ufb04", + "ffr;": "\U0001d523", + "filig;": "\ufb01", + "fjlig;": "fj", + "flat;": "\u266d", + "fllig;": "\ufb02", + "fltns;": "\u25b1", + "fnof;": "\u0192", + "fopf;": "\U0001d557", + "forall;": "\u2200", + "fork;": "\u22d4", + "forkv;": "\u2ad9", + "fpartint;": "\u2a0d", + "frac12": "\xbd", + "frac12;": "\xbd", + "frac13;": "\u2153", + "frac14": "\xbc", + "frac14;": "\xbc", + "frac15;": "\u2155", + "frac16;": "\u2159", + "frac18;": "\u215b", + "frac23;": "\u2154", + "frac25;": "\u2156", + "frac34": "\xbe", + "frac34;": "\xbe", + "frac35;": "\u2157", + "frac38;": "\u215c", + "frac45;": "\u2158", + "frac56;": "\u215a", + "frac58;": "\u215d", + "frac78;": "\u215e", + "frasl;": "\u2044", + "frown;": "\u2322", + "fscr;": "\U0001d4bb", + "gE;": "\u2267", + "gEl;": "\u2a8c", + "gacute;": "\u01f5", + "gamma;": "\u03b3", + "gammad;": "\u03dd", + "gap;": "\u2a86", + "gbreve;": "\u011f", + "gcirc;": "\u011d", + "gcy;": "\u0433", + "gdot;": "\u0121", + "ge;": "\u2265", + "gel;": "\u22db", + "geq;": "\u2265", + "geqq;": "\u2267", + "geqslant;": "\u2a7e", + "ges;": "\u2a7e", + "gescc;": "\u2aa9", + "gesdot;": "\u2a80", + "gesdoto;": "\u2a82", + "gesdotol;": "\u2a84", + "gesl;": "\u22db\ufe00", + "gesles;": "\u2a94", + "gfr;": "\U0001d524", + "gg;": "\u226b", + "ggg;": "\u22d9", + "gimel;": "\u2137", + "gjcy;": "\u0453", + "gl;": "\u2277", + "glE;": "\u2a92", + "gla;": "\u2aa5", + "glj;": "\u2aa4", + "gnE;": "\u2269", + "gnap;": "\u2a8a", + "gnapprox;": "\u2a8a", + "gne;": "\u2a88", + "gneq;": "\u2a88", + "gneqq;": "\u2269", + "gnsim;": "\u22e7", + "gopf;": "\U0001d558", + "grave;": "`", + "gscr;": "\u210a", + "gsim;": "\u2273", + "gsime;": "\u2a8e", + "gsiml;": "\u2a90", + "gt": ">", + "gt;": ">", + "gtcc;": "\u2aa7", + "gtcir;": "\u2a7a", + "gtdot;": "\u22d7", + "gtlPar;": "\u2995", + "gtquest;": "\u2a7c", + "gtrapprox;": "\u2a86", + "gtrarr;": "\u2978", + "gtrdot;": "\u22d7", + "gtreqless;": "\u22db", + "gtreqqless;": "\u2a8c", + "gtrless;": "\u2277", + "gtrsim;": "\u2273", + "gvertneqq;": "\u2269\ufe00", + "gvnE;": "\u2269\ufe00", + "hArr;": "\u21d4", + "hairsp;": "\u200a", + "half;": "\xbd", + "hamilt;": "\u210b", + "hardcy;": "\u044a", + "harr;": "\u2194", + "harrcir;": "\u2948", + "harrw;": "\u21ad", + "hbar;": "\u210f", + "hcirc;": "\u0125", + "hearts;": "\u2665", + "heartsuit;": "\u2665", + "hellip;": "\u2026", + "hercon;": "\u22b9", + "hfr;": "\U0001d525", + "hksearow;": "\u2925", + "hkswarow;": "\u2926", + "hoarr;": "\u21ff", + "homtht;": "\u223b", + "hookleftarrow;": "\u21a9", + "hookrightarrow;": "\u21aa", + "hopf;": "\U0001d559", + "horbar;": "\u2015", + "hscr;": "\U0001d4bd", + "hslash;": "\u210f", + "hstrok;": "\u0127", + "hybull;": "\u2043", + "hyphen;": "\u2010", + "iacute": "\xed", + "iacute;": "\xed", + "ic;": "\u2063", + "icirc": "\xee", + "icirc;": "\xee", + "icy;": "\u0438", + "iecy;": "\u0435", + "iexcl": "\xa1", + "iexcl;": "\xa1", + "iff;": "\u21d4", + "ifr;": "\U0001d526", + "igrave": "\xec", + "igrave;": "\xec", + "ii;": "\u2148", + "iiiint;": "\u2a0c", + "iiint;": "\u222d", + "iinfin;": "\u29dc", + "iiota;": "\u2129", + "ijlig;": "\u0133", + "imacr;": "\u012b", + "image;": "\u2111", + "imagline;": "\u2110", + "imagpart;": "\u2111", + "imath;": "\u0131", + "imof;": "\u22b7", + "imped;": "\u01b5", + "in;": "\u2208", + "incare;": "\u2105", + "infin;": "\u221e", + "infintie;": "\u29dd", + "inodot;": "\u0131", + "int;": "\u222b", + "intcal;": "\u22ba", + "integers;": "\u2124", + "intercal;": "\u22ba", + "intlarhk;": "\u2a17", + "intprod;": "\u2a3c", + "iocy;": "\u0451", + "iogon;": "\u012f", + "iopf;": "\U0001d55a", + "iota;": "\u03b9", + "iprod;": "\u2a3c", + "iquest": "\xbf", + "iquest;": "\xbf", + "iscr;": "\U0001d4be", + "isin;": "\u2208", + "isinE;": "\u22f9", + "isindot;": "\u22f5", + "isins;": "\u22f4", + "isinsv;": "\u22f3", + "isinv;": "\u2208", + "it;": "\u2062", + "itilde;": "\u0129", + "iukcy;": "\u0456", + "iuml": "\xef", + "iuml;": "\xef", + "jcirc;": "\u0135", + "jcy;": "\u0439", + "jfr;": "\U0001d527", + "jmath;": "\u0237", + "jopf;": "\U0001d55b", + "jscr;": "\U0001d4bf", + "jsercy;": "\u0458", + "jukcy;": "\u0454", + "kappa;": "\u03ba", + "kappav;": "\u03f0", + "kcedil;": "\u0137", + "kcy;": "\u043a", + "kfr;": "\U0001d528", + "kgreen;": "\u0138", + "khcy;": "\u0445", + "kjcy;": "\u045c", + "kopf;": "\U0001d55c", + "kscr;": "\U0001d4c0", + "lAarr;": "\u21da", + "lArr;": "\u21d0", + "lAtail;": "\u291b", + "lBarr;": "\u290e", + "lE;": "\u2266", + "lEg;": "\u2a8b", + "lHar;": "\u2962", + "lacute;": "\u013a", + "laemptyv;": "\u29b4", + "lagran;": "\u2112", + "lambda;": "\u03bb", + "lang;": "\u27e8", + "langd;": "\u2991", + "langle;": "\u27e8", + "lap;": "\u2a85", + "laquo": "\xab", + "laquo;": "\xab", + "larr;": "\u2190", + "larrb;": "\u21e4", + "larrbfs;": "\u291f", + "larrfs;": "\u291d", + "larrhk;": "\u21a9", + "larrlp;": "\u21ab", + "larrpl;": "\u2939", + "larrsim;": "\u2973", + "larrtl;": "\u21a2", + "lat;": "\u2aab", + "latail;": "\u2919", + "late;": "\u2aad", + "lates;": "\u2aad\ufe00", + "lbarr;": "\u290c", + "lbbrk;": "\u2772", + "lbrace;": "{", + "lbrack;": "[", + "lbrke;": "\u298b", + "lbrksld;": "\u298f", + "lbrkslu;": "\u298d", + "lcaron;": "\u013e", + "lcedil;": "\u013c", + "lceil;": "\u2308", + "lcub;": "{", + "lcy;": "\u043b", + "ldca;": "\u2936", + "ldquo;": "\u201c", + "ldquor;": "\u201e", + "ldrdhar;": "\u2967", + "ldrushar;": "\u294b", + "ldsh;": "\u21b2", + "le;": "\u2264", + "leftarrow;": "\u2190", + "leftarrowtail;": "\u21a2", + "leftharpoondown;": "\u21bd", + "leftharpoonup;": "\u21bc", + "leftleftarrows;": "\u21c7", + "leftrightarrow;": "\u2194", + "leftrightarrows;": "\u21c6", + "leftrightharpoons;": "\u21cb", + "leftrightsquigarrow;": "\u21ad", + "leftthreetimes;": "\u22cb", + "leg;": "\u22da", + "leq;": "\u2264", + "leqq;": "\u2266", + "leqslant;": "\u2a7d", + "les;": "\u2a7d", + "lescc;": "\u2aa8", + "lesdot;": "\u2a7f", + "lesdoto;": "\u2a81", + "lesdotor;": "\u2a83", + "lesg;": "\u22da\ufe00", + "lesges;": "\u2a93", + "lessapprox;": "\u2a85", + "lessdot;": "\u22d6", + "lesseqgtr;": "\u22da", + "lesseqqgtr;": "\u2a8b", + "lessgtr;": "\u2276", + "lesssim;": "\u2272", + "lfisht;": "\u297c", + "lfloor;": "\u230a", + "lfr;": "\U0001d529", + "lg;": "\u2276", + "lgE;": "\u2a91", + "lhard;": "\u21bd", + "lharu;": "\u21bc", + "lharul;": "\u296a", + "lhblk;": "\u2584", + "ljcy;": "\u0459", + "ll;": "\u226a", + "llarr;": "\u21c7", + "llcorner;": "\u231e", + "llhard;": "\u296b", + "lltri;": "\u25fa", + "lmidot;": "\u0140", + "lmoust;": "\u23b0", + "lmoustache;": "\u23b0", + "lnE;": "\u2268", + "lnap;": "\u2a89", + "lnapprox;": "\u2a89", + "lne;": "\u2a87", + "lneq;": "\u2a87", + "lneqq;": "\u2268", + "lnsim;": "\u22e6", + "loang;": "\u27ec", + "loarr;": "\u21fd", + "lobrk;": "\u27e6", + "longleftarrow;": "\u27f5", + "longleftrightarrow;": "\u27f7", + "longmapsto;": "\u27fc", + "longrightarrow;": "\u27f6", + "looparrowleft;": "\u21ab", + "looparrowright;": "\u21ac", + "lopar;": "\u2985", + "lopf;": "\U0001d55d", + "loplus;": "\u2a2d", + "lotimes;": "\u2a34", + "lowast;": "\u2217", + "lowbar;": "_", + "loz;": "\u25ca", + "lozenge;": "\u25ca", + "lozf;": "\u29eb", + "lpar;": "(", + "lparlt;": "\u2993", + "lrarr;": "\u21c6", + "lrcorner;": "\u231f", + "lrhar;": "\u21cb", + "lrhard;": "\u296d", + "lrm;": "\u200e", + "lrtri;": "\u22bf", + "lsaquo;": "\u2039", + "lscr;": "\U0001d4c1", + "lsh;": "\u21b0", + "lsim;": "\u2272", + "lsime;": "\u2a8d", + "lsimg;": "\u2a8f", + "lsqb;": "[", + "lsquo;": "\u2018", + "lsquor;": "\u201a", + "lstrok;": "\u0142", + "lt": "<", + "lt;": "<", + "ltcc;": "\u2aa6", + "ltcir;": "\u2a79", + "ltdot;": "\u22d6", + "lthree;": "\u22cb", + "ltimes;": "\u22c9", + "ltlarr;": "\u2976", + "ltquest;": "\u2a7b", + "ltrPar;": "\u2996", + "ltri;": "\u25c3", + "ltrie;": "\u22b4", + "ltrif;": "\u25c2", + "lurdshar;": "\u294a", + "luruhar;": "\u2966", + "lvertneqq;": "\u2268\ufe00", + "lvnE;": "\u2268\ufe00", + "mDDot;": "\u223a", + "macr": "\xaf", + "macr;": "\xaf", + "male;": "\u2642", + "malt;": "\u2720", + "maltese;": "\u2720", + "map;": "\u21a6", + "mapsto;": "\u21a6", + "mapstodown;": "\u21a7", + "mapstoleft;": "\u21a4", + "mapstoup;": "\u21a5", + "marker;": "\u25ae", + "mcomma;": "\u2a29", + "mcy;": "\u043c", + "mdash;": "\u2014", + "measuredangle;": "\u2221", + "mfr;": "\U0001d52a", + "mho;": "\u2127", + "micro": "\xb5", + "micro;": "\xb5", + "mid;": "\u2223", + "midast;": "*", + "midcir;": "\u2af0", + "middot": "\xb7", + "middot;": "\xb7", + "minus;": "\u2212", + "minusb;": "\u229f", + "minusd;": "\u2238", + "minusdu;": "\u2a2a", + "mlcp;": "\u2adb", + "mldr;": "\u2026", + "mnplus;": "\u2213", + "models;": "\u22a7", + "mopf;": "\U0001d55e", + "mp;": "\u2213", + "mscr;": "\U0001d4c2", + "mstpos;": "\u223e", + "mu;": "\u03bc", + "multimap;": "\u22b8", + "mumap;": "\u22b8", + "nGg;": "\u22d9\u0338", + "nGt;": "\u226b\u20d2", + "nGtv;": "\u226b\u0338", + "nLeftarrow;": "\u21cd", + "nLeftrightarrow;": "\u21ce", + "nLl;": "\u22d8\u0338", + "nLt;": "\u226a\u20d2", + "nLtv;": "\u226a\u0338", + "nRightarrow;": "\u21cf", + "nVDash;": "\u22af", + "nVdash;": "\u22ae", + "nabla;": "\u2207", + "nacute;": "\u0144", + "nang;": "\u2220\u20d2", + "nap;": "\u2249", + "napE;": "\u2a70\u0338", + "napid;": "\u224b\u0338", + "napos;": "\u0149", + "napprox;": "\u2249", + "natur;": "\u266e", + "natural;": "\u266e", + "naturals;": "\u2115", + "nbsp": "\xa0", + "nbsp;": "\xa0", + "nbump;": "\u224e\u0338", + "nbumpe;": "\u224f\u0338", + "ncap;": "\u2a43", + "ncaron;": "\u0148", + "ncedil;": "\u0146", + "ncong;": "\u2247", + "ncongdot;": "\u2a6d\u0338", + "ncup;": "\u2a42", + "ncy;": "\u043d", + "ndash;": "\u2013", + "ne;": "\u2260", + "neArr;": "\u21d7", + "nearhk;": "\u2924", + "nearr;": "\u2197", + "nearrow;": "\u2197", + "nedot;": "\u2250\u0338", + "nequiv;": "\u2262", + "nesear;": "\u2928", + "nesim;": "\u2242\u0338", + "nexist;": "\u2204", + "nexists;": "\u2204", + "nfr;": "\U0001d52b", + "ngE;": "\u2267\u0338", + "nge;": "\u2271", + "ngeq;": "\u2271", + "ngeqq;": "\u2267\u0338", + "ngeqslant;": "\u2a7e\u0338", + "nges;": "\u2a7e\u0338", + "ngsim;": "\u2275", + "ngt;": "\u226f", + "ngtr;": "\u226f", + "nhArr;": "\u21ce", + "nharr;": "\u21ae", + "nhpar;": "\u2af2", + "ni;": "\u220b", + "nis;": "\u22fc", + "nisd;": "\u22fa", + "niv;": "\u220b", + "njcy;": "\u045a", + "nlArr;": "\u21cd", + "nlE;": "\u2266\u0338", + "nlarr;": "\u219a", + "nldr;": "\u2025", + "nle;": "\u2270", + "nleftarrow;": "\u219a", + "nleftrightarrow;": "\u21ae", + "nleq;": "\u2270", + "nleqq;": "\u2266\u0338", + "nleqslant;": "\u2a7d\u0338", + "nles;": "\u2a7d\u0338", + "nless;": "\u226e", + "nlsim;": "\u2274", + "nlt;": "\u226e", + "nltri;": "\u22ea", + "nltrie;": "\u22ec", + "nmid;": "\u2224", + "nopf;": "\U0001d55f", + "not": "\xac", + "not;": "\xac", + "notin;": "\u2209", + "notinE;": "\u22f9\u0338", + "notindot;": "\u22f5\u0338", + "notinva;": "\u2209", + "notinvb;": "\u22f7", + "notinvc;": "\u22f6", + "notni;": "\u220c", + "notniva;": "\u220c", + "notnivb;": "\u22fe", + "notnivc;": "\u22fd", + "npar;": "\u2226", + "nparallel;": "\u2226", + "nparsl;": "\u2afd\u20e5", + "npart;": "\u2202\u0338", + "npolint;": "\u2a14", + "npr;": "\u2280", + "nprcue;": "\u22e0", + "npre;": "\u2aaf\u0338", + "nprec;": "\u2280", + "npreceq;": "\u2aaf\u0338", + "nrArr;": "\u21cf", + "nrarr;": "\u219b", + "nrarrc;": "\u2933\u0338", + "nrarrw;": "\u219d\u0338", + "nrightarrow;": "\u219b", + "nrtri;": "\u22eb", + "nrtrie;": "\u22ed", + "nsc;": "\u2281", + "nsccue;": "\u22e1", + "nsce;": "\u2ab0\u0338", + "nscr;": "\U0001d4c3", + "nshortmid;": "\u2224", + "nshortparallel;": "\u2226", + "nsim;": "\u2241", + "nsime;": "\u2244", + "nsimeq;": "\u2244", + "nsmid;": "\u2224", + "nspar;": "\u2226", + "nsqsube;": "\u22e2", + "nsqsupe;": "\u22e3", + "nsub;": "\u2284", + "nsubE;": "\u2ac5\u0338", + "nsube;": "\u2288", + "nsubset;": "\u2282\u20d2", + "nsubseteq;": "\u2288", + "nsubseteqq;": "\u2ac5\u0338", + "nsucc;": "\u2281", + "nsucceq;": "\u2ab0\u0338", + "nsup;": "\u2285", + "nsupE;": "\u2ac6\u0338", + "nsupe;": "\u2289", + "nsupset;": "\u2283\u20d2", + "nsupseteq;": "\u2289", + "nsupseteqq;": "\u2ac6\u0338", + "ntgl;": "\u2279", + "ntilde": "\xf1", + "ntilde;": "\xf1", + "ntlg;": "\u2278", + "ntriangleleft;": "\u22ea", + "ntrianglelefteq;": "\u22ec", + "ntriangleright;": "\u22eb", + "ntrianglerighteq;": "\u22ed", + "nu;": "\u03bd", + "num;": "#", + "numero;": "\u2116", + "numsp;": "\u2007", + "nvDash;": "\u22ad", + "nvHarr;": "\u2904", + "nvap;": "\u224d\u20d2", + "nvdash;": "\u22ac", + "nvge;": "\u2265\u20d2", + "nvgt;": ">\u20d2", + "nvinfin;": "\u29de", + "nvlArr;": "\u2902", + "nvle;": "\u2264\u20d2", + "nvlt;": "<\u20d2", + "nvltrie;": "\u22b4\u20d2", + "nvrArr;": "\u2903", + "nvrtrie;": "\u22b5\u20d2", + "nvsim;": "\u223c\u20d2", + "nwArr;": "\u21d6", + "nwarhk;": "\u2923", + "nwarr;": "\u2196", + "nwarrow;": "\u2196", + "nwnear;": "\u2927", + "oS;": "\u24c8", + "oacute": "\xf3", + "oacute;": "\xf3", + "oast;": "\u229b", + "ocir;": "\u229a", + "ocirc": "\xf4", + "ocirc;": "\xf4", + "ocy;": "\u043e", + "odash;": "\u229d", + "odblac;": "\u0151", + "odiv;": "\u2a38", + "odot;": "\u2299", + "odsold;": "\u29bc", + "oelig;": "\u0153", + "ofcir;": "\u29bf", + "ofr;": "\U0001d52c", + "ogon;": "\u02db", + "ograve": "\xf2", + "ograve;": "\xf2", + "ogt;": "\u29c1", + "ohbar;": "\u29b5", + "ohm;": "\u03a9", + "oint;": "\u222e", + "olarr;": "\u21ba", + "olcir;": "\u29be", + "olcross;": "\u29bb", + "oline;": "\u203e", + "olt;": "\u29c0", + "omacr;": "\u014d", + "omega;": "\u03c9", + "omicron;": "\u03bf", + "omid;": "\u29b6", + "ominus;": "\u2296", + "oopf;": "\U0001d560", + "opar;": "\u29b7", + "operp;": "\u29b9", + "oplus;": "\u2295", + "or;": "\u2228", + "orarr;": "\u21bb", + "ord;": "\u2a5d", + "order;": "\u2134", + "orderof;": "\u2134", + "ordf": "\xaa", + "ordf;": "\xaa", + "ordm": "\xba", + "ordm;": "\xba", + "origof;": "\u22b6", + "oror;": "\u2a56", + "orslope;": "\u2a57", + "orv;": "\u2a5b", + "oscr;": "\u2134", + "oslash": "\xf8", + "oslash;": "\xf8", + "osol;": "\u2298", + "otilde": "\xf5", + "otilde;": "\xf5", + "otimes;": "\u2297", + "otimesas;": "\u2a36", + "ouml": "\xf6", + "ouml;": "\xf6", + "ovbar;": "\u233d", + "par;": "\u2225", + "para": "\xb6", + "para;": "\xb6", + "parallel;": "\u2225", + "parsim;": "\u2af3", + "parsl;": "\u2afd", + "part;": "\u2202", + "pcy;": "\u043f", + "percnt;": "%", + "period;": ".", + "permil;": "\u2030", + "perp;": "\u22a5", + "pertenk;": "\u2031", + "pfr;": "\U0001d52d", + "phi;": "\u03c6", + "phiv;": "\u03d5", + "phmmat;": "\u2133", + "phone;": "\u260e", + "pi;": "\u03c0", + "pitchfork;": "\u22d4", + "piv;": "\u03d6", + "planck;": "\u210f", + "planckh;": "\u210e", + "plankv;": "\u210f", + "plus;": "+", + "plusacir;": "\u2a23", + "plusb;": "\u229e", + "pluscir;": "\u2a22", + "plusdo;": "\u2214", + "plusdu;": "\u2a25", + "pluse;": "\u2a72", + "plusmn": "\xb1", + "plusmn;": "\xb1", + "plussim;": "\u2a26", + "plustwo;": "\u2a27", + "pm;": "\xb1", + "pointint;": "\u2a15", + "popf;": "\U0001d561", + "pound": "\xa3", + "pound;": "\xa3", + "pr;": "\u227a", + "prE;": "\u2ab3", + "prap;": "\u2ab7", + "prcue;": "\u227c", + "pre;": "\u2aaf", + "prec;": "\u227a", + "precapprox;": "\u2ab7", + "preccurlyeq;": "\u227c", + "preceq;": "\u2aaf", + "precnapprox;": "\u2ab9", + "precneqq;": "\u2ab5", + "precnsim;": "\u22e8", + "precsim;": "\u227e", + "prime;": "\u2032", + "primes;": "\u2119", + "prnE;": "\u2ab5", + "prnap;": "\u2ab9", + "prnsim;": "\u22e8", + "prod;": "\u220f", + "profalar;": "\u232e", + "profline;": "\u2312", + "profsurf;": "\u2313", + "prop;": "\u221d", + "propto;": "\u221d", + "prsim;": "\u227e", + "prurel;": "\u22b0", + "pscr;": "\U0001d4c5", + "psi;": "\u03c8", + "puncsp;": "\u2008", + "qfr;": "\U0001d52e", + "qint;": "\u2a0c", + "qopf;": "\U0001d562", + "qprime;": "\u2057", + "qscr;": "\U0001d4c6", + "quaternions;": "\u210d", + "quatint;": "\u2a16", + "quest;": "?", + "questeq;": "\u225f", + "quot": "\"", + "quot;": "\"", + "rAarr;": "\u21db", + "rArr;": "\u21d2", + "rAtail;": "\u291c", + "rBarr;": "\u290f", + "rHar;": "\u2964", + "race;": "\u223d\u0331", + "racute;": "\u0155", + "radic;": "\u221a", + "raemptyv;": "\u29b3", + "rang;": "\u27e9", + "rangd;": "\u2992", + "range;": "\u29a5", + "rangle;": "\u27e9", + "raquo": "\xbb", + "raquo;": "\xbb", + "rarr;": "\u2192", + "rarrap;": "\u2975", + "rarrb;": "\u21e5", + "rarrbfs;": "\u2920", + "rarrc;": "\u2933", + "rarrfs;": "\u291e", + "rarrhk;": "\u21aa", + "rarrlp;": "\u21ac", + "rarrpl;": "\u2945", + "rarrsim;": "\u2974", + "rarrtl;": "\u21a3", + "rarrw;": "\u219d", + "ratail;": "\u291a", + "ratio;": "\u2236", + "rationals;": "\u211a", + "rbarr;": "\u290d", + "rbbrk;": "\u2773", + "rbrace;": "}", + "rbrack;": "]", + "rbrke;": "\u298c", + "rbrksld;": "\u298e", + "rbrkslu;": "\u2990", + "rcaron;": "\u0159", + "rcedil;": "\u0157", + "rceil;": "\u2309", + "rcub;": "}", + "rcy;": "\u0440", + "rdca;": "\u2937", + "rdldhar;": "\u2969", + "rdquo;": "\u201d", + "rdquor;": "\u201d", + "rdsh;": "\u21b3", + "real;": "\u211c", + "realine;": "\u211b", + "realpart;": "\u211c", + "reals;": "\u211d", + "rect;": "\u25ad", + "reg": "\xae", + "reg;": "\xae", + "rfisht;": "\u297d", + "rfloor;": "\u230b", + "rfr;": "\U0001d52f", + "rhard;": "\u21c1", + "rharu;": "\u21c0", + "rharul;": "\u296c", + "rho;": "\u03c1", + "rhov;": "\u03f1", + "rightarrow;": "\u2192", + "rightarrowtail;": "\u21a3", + "rightharpoondown;": "\u21c1", + "rightharpoonup;": "\u21c0", + "rightleftarrows;": "\u21c4", + "rightleftharpoons;": "\u21cc", + "rightrightarrows;": "\u21c9", + "rightsquigarrow;": "\u219d", + "rightthreetimes;": "\u22cc", + "ring;": "\u02da", + "risingdotseq;": "\u2253", + "rlarr;": "\u21c4", + "rlhar;": "\u21cc", + "rlm;": "\u200f", + "rmoust;": "\u23b1", + "rmoustache;": "\u23b1", + "rnmid;": "\u2aee", + "roang;": "\u27ed", + "roarr;": "\u21fe", + "robrk;": "\u27e7", + "ropar;": "\u2986", + "ropf;": "\U0001d563", + "roplus;": "\u2a2e", + "rotimes;": "\u2a35", + "rpar;": ")", + "rpargt;": "\u2994", + "rppolint;": "\u2a12", + "rrarr;": "\u21c9", + "rsaquo;": "\u203a", + "rscr;": "\U0001d4c7", + "rsh;": "\u21b1", + "rsqb;": "]", + "rsquo;": "\u2019", + "rsquor;": "\u2019", + "rthree;": "\u22cc", + "rtimes;": "\u22ca", + "rtri;": "\u25b9", + "rtrie;": "\u22b5", + "rtrif;": "\u25b8", + "rtriltri;": "\u29ce", + "ruluhar;": "\u2968", + "rx;": "\u211e", + "sacute;": "\u015b", + "sbquo;": "\u201a", + "sc;": "\u227b", + "scE;": "\u2ab4", + "scap;": "\u2ab8", + "scaron;": "\u0161", + "sccue;": "\u227d", + "sce;": "\u2ab0", + "scedil;": "\u015f", + "scirc;": "\u015d", + "scnE;": "\u2ab6", + "scnap;": "\u2aba", + "scnsim;": "\u22e9", + "scpolint;": "\u2a13", + "scsim;": "\u227f", + "scy;": "\u0441", + "sdot;": "\u22c5", + "sdotb;": "\u22a1", + "sdote;": "\u2a66", + "seArr;": "\u21d8", + "searhk;": "\u2925", + "searr;": "\u2198", + "searrow;": "\u2198", + "sect": "\xa7", + "sect;": "\xa7", + "semi;": ";", + "seswar;": "\u2929", + "setminus;": "\u2216", + "setmn;": "\u2216", + "sext;": "\u2736", + "sfr;": "\U0001d530", + "sfrown;": "\u2322", + "sharp;": "\u266f", + "shchcy;": "\u0449", + "shcy;": "\u0448", + "shortmid;": "\u2223", + "shortparallel;": "\u2225", + "shy": "\xad", + "shy;": "\xad", + "sigma;": "\u03c3", + "sigmaf;": "\u03c2", + "sigmav;": "\u03c2", + "sim;": "\u223c", + "simdot;": "\u2a6a", + "sime;": "\u2243", + "simeq;": "\u2243", + "simg;": "\u2a9e", + "simgE;": "\u2aa0", + "siml;": "\u2a9d", + "simlE;": "\u2a9f", + "simne;": "\u2246", + "simplus;": "\u2a24", + "simrarr;": "\u2972", + "slarr;": "\u2190", + "smallsetminus;": "\u2216", + "smashp;": "\u2a33", + "smeparsl;": "\u29e4", + "smid;": "\u2223", + "smile;": "\u2323", + "smt;": "\u2aaa", + "smte;": "\u2aac", + "smtes;": "\u2aac\ufe00", + "softcy;": "\u044c", + "sol;": "/", + "solb;": "\u29c4", + "solbar;": "\u233f", + "sopf;": "\U0001d564", + "spades;": "\u2660", + "spadesuit;": "\u2660", + "spar;": "\u2225", + "sqcap;": "\u2293", + "sqcaps;": "\u2293\ufe00", + "sqcup;": "\u2294", + "sqcups;": "\u2294\ufe00", + "sqsub;": "\u228f", + "sqsube;": "\u2291", + "sqsubset;": "\u228f", + "sqsubseteq;": "\u2291", + "sqsup;": "\u2290", + "sqsupe;": "\u2292", + "sqsupset;": "\u2290", + "sqsupseteq;": "\u2292", + "squ;": "\u25a1", + "square;": "\u25a1", + "squarf;": "\u25aa", + "squf;": "\u25aa", + "srarr;": "\u2192", + "sscr;": "\U0001d4c8", + "ssetmn;": "\u2216", + "ssmile;": "\u2323", + "sstarf;": "\u22c6", + "star;": "\u2606", + "starf;": "\u2605", + "straightepsilon;": "\u03f5", + "straightphi;": "\u03d5", + "strns;": "\xaf", + "sub;": "\u2282", + "subE;": "\u2ac5", + "subdot;": "\u2abd", + "sube;": "\u2286", + "subedot;": "\u2ac3", + "submult;": "\u2ac1", + "subnE;": "\u2acb", + "subne;": "\u228a", + "subplus;": "\u2abf", + "subrarr;": "\u2979", + "subset;": "\u2282", + "subseteq;": "\u2286", + "subseteqq;": "\u2ac5", + "subsetneq;": "\u228a", + "subsetneqq;": "\u2acb", + "subsim;": "\u2ac7", + "subsub;": "\u2ad5", + "subsup;": "\u2ad3", + "succ;": "\u227b", + "succapprox;": "\u2ab8", + "succcurlyeq;": "\u227d", + "succeq;": "\u2ab0", + "succnapprox;": "\u2aba", + "succneqq;": "\u2ab6", + "succnsim;": "\u22e9", + "succsim;": "\u227f", + "sum;": "\u2211", + "sung;": "\u266a", + "sup1": "\xb9", + "sup1;": "\xb9", + "sup2": "\xb2", + "sup2;": "\xb2", + "sup3": "\xb3", + "sup3;": "\xb3", + "sup;": "\u2283", + "supE;": "\u2ac6", + "supdot;": "\u2abe", + "supdsub;": "\u2ad8", + "supe;": "\u2287", + "supedot;": "\u2ac4", + "suphsol;": "\u27c9", + "suphsub;": "\u2ad7", + "suplarr;": "\u297b", + "supmult;": "\u2ac2", + "supnE;": "\u2acc", + "supne;": "\u228b", + "supplus;": "\u2ac0", + "supset;": "\u2283", + "supseteq;": "\u2287", + "supseteqq;": "\u2ac6", + "supsetneq;": "\u228b", + "supsetneqq;": "\u2acc", + "supsim;": "\u2ac8", + "supsub;": "\u2ad4", + "supsup;": "\u2ad6", + "swArr;": "\u21d9", + "swarhk;": "\u2926", + "swarr;": "\u2199", + "swarrow;": "\u2199", + "swnwar;": "\u292a", + "szlig": "\xdf", + "szlig;": "\xdf", + "target;": "\u2316", + "tau;": "\u03c4", + "tbrk;": "\u23b4", + "tcaron;": "\u0165", + "tcedil;": "\u0163", + "tcy;": "\u0442", + "tdot;": "\u20db", + "telrec;": "\u2315", + "tfr;": "\U0001d531", + "there4;": "\u2234", + "therefore;": "\u2234", + "theta;": "\u03b8", + "thetasym;": "\u03d1", + "thetav;": "\u03d1", + "thickapprox;": "\u2248", + "thicksim;": "\u223c", + "thinsp;": "\u2009", + "thkap;": "\u2248", + "thksim;": "\u223c", + "thorn": "\xfe", + "thorn;": "\xfe", + "tilde;": "\u02dc", + "times": "\xd7", + "times;": "\xd7", + "timesb;": "\u22a0", + "timesbar;": "\u2a31", + "timesd;": "\u2a30", + "tint;": "\u222d", + "toea;": "\u2928", + "top;": "\u22a4", + "topbot;": "\u2336", + "topcir;": "\u2af1", + "topf;": "\U0001d565", + "topfork;": "\u2ada", + "tosa;": "\u2929", + "tprime;": "\u2034", + "trade;": "\u2122", + "triangle;": "\u25b5", + "triangledown;": "\u25bf", + "triangleleft;": "\u25c3", + "trianglelefteq;": "\u22b4", + "triangleq;": "\u225c", + "triangleright;": "\u25b9", + "trianglerighteq;": "\u22b5", + "tridot;": "\u25ec", + "trie;": "\u225c", + "triminus;": "\u2a3a", + "triplus;": "\u2a39", + "trisb;": "\u29cd", + "tritime;": "\u2a3b", + "trpezium;": "\u23e2", + "tscr;": "\U0001d4c9", + "tscy;": "\u0446", + "tshcy;": "\u045b", + "tstrok;": "\u0167", + "twixt;": "\u226c", + "twoheadleftarrow;": "\u219e", + "twoheadrightarrow;": "\u21a0", + "uArr;": "\u21d1", + "uHar;": "\u2963", + "uacute": "\xfa", + "uacute;": "\xfa", + "uarr;": "\u2191", + "ubrcy;": "\u045e", + "ubreve;": "\u016d", + "ucirc": "\xfb", + "ucirc;": "\xfb", + "ucy;": "\u0443", + "udarr;": "\u21c5", + "udblac;": "\u0171", + "udhar;": "\u296e", + "ufisht;": "\u297e", + "ufr;": "\U0001d532", + "ugrave": "\xf9", + "ugrave;": "\xf9", + "uharl;": "\u21bf", + "uharr;": "\u21be", + "uhblk;": "\u2580", + "ulcorn;": "\u231c", + "ulcorner;": "\u231c", + "ulcrop;": "\u230f", + "ultri;": "\u25f8", + "umacr;": "\u016b", + "uml": "\xa8", + "uml;": "\xa8", + "uogon;": "\u0173", + "uopf;": "\U0001d566", + "uparrow;": "\u2191", + "updownarrow;": "\u2195", + "upharpoonleft;": "\u21bf", + "upharpoonright;": "\u21be", + "uplus;": "\u228e", + "upsi;": "\u03c5", + "upsih;": "\u03d2", + "upsilon;": "\u03c5", + "upuparrows;": "\u21c8", + "urcorn;": "\u231d", + "urcorner;": "\u231d", + "urcrop;": "\u230e", + "uring;": "\u016f", + "urtri;": "\u25f9", + "uscr;": "\U0001d4ca", + "utdot;": "\u22f0", + "utilde;": "\u0169", + "utri;": "\u25b5", + "utrif;": "\u25b4", + "uuarr;": "\u21c8", + "uuml": "\xfc", + "uuml;": "\xfc", + "uwangle;": "\u29a7", + "vArr;": "\u21d5", + "vBar;": "\u2ae8", + "vBarv;": "\u2ae9", + "vDash;": "\u22a8", + "vangrt;": "\u299c", + "varepsilon;": "\u03f5", + "varkappa;": "\u03f0", + "varnothing;": "\u2205", + "varphi;": "\u03d5", + "varpi;": "\u03d6", + "varpropto;": "\u221d", + "varr;": "\u2195", + "varrho;": "\u03f1", + "varsigma;": "\u03c2", + "varsubsetneq;": "\u228a\ufe00", + "varsubsetneqq;": "\u2acb\ufe00", + "varsupsetneq;": "\u228b\ufe00", + "varsupsetneqq;": "\u2acc\ufe00", + "vartheta;": "\u03d1", + "vartriangleleft;": "\u22b2", + "vartriangleright;": "\u22b3", + "vcy;": "\u0432", + "vdash;": "\u22a2", + "vee;": "\u2228", + "veebar;": "\u22bb", + "veeeq;": "\u225a", + "vellip;": "\u22ee", + "verbar;": "|", + "vert;": "|", + "vfr;": "\U0001d533", + "vltri;": "\u22b2", + "vnsub;": "\u2282\u20d2", + "vnsup;": "\u2283\u20d2", + "vopf;": "\U0001d567", + "vprop;": "\u221d", + "vrtri;": "\u22b3", + "vscr;": "\U0001d4cb", + "vsubnE;": "\u2acb\ufe00", + "vsubne;": "\u228a\ufe00", + "vsupnE;": "\u2acc\ufe00", + "vsupne;": "\u228b\ufe00", + "vzigzag;": "\u299a", + "wcirc;": "\u0175", + "wedbar;": "\u2a5f", + "wedge;": "\u2227", + "wedgeq;": "\u2259", + "weierp;": "\u2118", + "wfr;": "\U0001d534", + "wopf;": "\U0001d568", + "wp;": "\u2118", + "wr;": "\u2240", + "wreath;": "\u2240", + "wscr;": "\U0001d4cc", + "xcap;": "\u22c2", + "xcirc;": "\u25ef", + "xcup;": "\u22c3", + "xdtri;": "\u25bd", + "xfr;": "\U0001d535", + "xhArr;": "\u27fa", + "xharr;": "\u27f7", + "xi;": "\u03be", + "xlArr;": "\u27f8", + "xlarr;": "\u27f5", + "xmap;": "\u27fc", + "xnis;": "\u22fb", + "xodot;": "\u2a00", + "xopf;": "\U0001d569", + "xoplus;": "\u2a01", + "xotime;": "\u2a02", + "xrArr;": "\u27f9", + "xrarr;": "\u27f6", + "xscr;": "\U0001d4cd", + "xsqcup;": "\u2a06", + "xuplus;": "\u2a04", + "xutri;": "\u25b3", + "xvee;": "\u22c1", + "xwedge;": "\u22c0", + "yacute": "\xfd", + "yacute;": "\xfd", + "yacy;": "\u044f", + "ycirc;": "\u0177", + "ycy;": "\u044b", + "yen": "\xa5", + "yen;": "\xa5", + "yfr;": "\U0001d536", + "yicy;": "\u0457", + "yopf;": "\U0001d56a", + "yscr;": "\U0001d4ce", + "yucy;": "\u044e", + "yuml": "\xff", + "yuml;": "\xff", + "zacute;": "\u017a", + "zcaron;": "\u017e", + "zcy;": "\u0437", + "zdot;": "\u017c", + "zeetrf;": "\u2128", + "zeta;": "\u03b6", + "zfr;": "\U0001d537", + "zhcy;": "\u0436", + "zigrarr;": "\u21dd", + "zopf;": "\U0001d56b", + "zscr;": "\U0001d4cf", + "zwj;": "\u200d", + "zwnj;": "\u200c", +} + +replacementCharacters = { + 0x0: "\uFFFD", + 0x0d: "\u000D", + 0x80: "\u20AC", + 0x81: "\u0081", + 0x81: "\u0081", + 0x82: "\u201A", + 0x83: "\u0192", + 0x84: "\u201E", + 0x85: "\u2026", + 0x86: "\u2020", + 0x87: "\u2021", + 0x88: "\u02C6", + 0x89: "\u2030", + 0x8A: "\u0160", + 0x8B: "\u2039", + 0x8C: "\u0152", + 0x8D: "\u008D", + 0x8E: "\u017D", + 0x8F: "\u008F", + 0x90: "\u0090", + 0x91: "\u2018", + 0x92: "\u2019", + 0x93: "\u201C", + 0x94: "\u201D", + 0x95: "\u2022", + 0x96: "\u2013", + 0x97: "\u2014", + 0x98: "\u02DC", + 0x99: "\u2122", + 0x9A: "\u0161", + 0x9B: "\u203A", + 0x9C: "\u0153", + 0x9D: "\u009D", + 0x9E: "\u017E", + 0x9F: "\u0178", +} + +encodings = { + '437': 'cp437', + '850': 'cp850', + '852': 'cp852', + '855': 'cp855', + '857': 'cp857', + '860': 'cp860', + '861': 'cp861', + '862': 'cp862', + '863': 'cp863', + '865': 'cp865', + '866': 'cp866', + '869': 'cp869', + 'ansix341968': 'ascii', + 'ansix341986': 'ascii', + 'arabic': 'iso8859-6', + 'ascii': 'ascii', + 'asmo708': 'iso8859-6', + 'big5': 'big5', + 'big5hkscs': 'big5hkscs', + 'chinese': 'gbk', + 'cp037': 'cp037', + 'cp1026': 'cp1026', + 'cp154': 'ptcp154', + 'cp367': 'ascii', + 'cp424': 'cp424', + 'cp437': 'cp437', + 'cp500': 'cp500', + 'cp775': 'cp775', + 'cp819': 'windows-1252', + 'cp850': 'cp850', + 'cp852': 'cp852', + 'cp855': 'cp855', + 'cp857': 'cp857', + 'cp860': 'cp860', + 'cp861': 'cp861', + 'cp862': 'cp862', + 'cp863': 'cp863', + 'cp864': 'cp864', + 'cp865': 'cp865', + 'cp866': 'cp866', + 'cp869': 'cp869', + 'cp936': 'gbk', + 'cpgr': 'cp869', + 'cpis': 'cp861', + 'csascii': 'ascii', + 'csbig5': 'big5', + 'cseuckr': 'cp949', + 'cseucpkdfmtjapanese': 'euc_jp', + 'csgb2312': 'gbk', + 'cshproman8': 'hp-roman8', + 'csibm037': 'cp037', + 'csibm1026': 'cp1026', + 'csibm424': 'cp424', + 'csibm500': 'cp500', + 'csibm855': 'cp855', + 'csibm857': 'cp857', + 'csibm860': 'cp860', + 'csibm861': 'cp861', + 'csibm863': 'cp863', + 'csibm864': 'cp864', + 'csibm865': 'cp865', + 'csibm866': 'cp866', + 'csibm869': 'cp869', + 'csiso2022jp': 'iso2022_jp', + 'csiso2022jp2': 'iso2022_jp_2', + 'csiso2022kr': 'iso2022_kr', + 'csiso58gb231280': 'gbk', + 'csisolatin1': 'windows-1252', + 'csisolatin2': 'iso8859-2', + 'csisolatin3': 'iso8859-3', + 'csisolatin4': 'iso8859-4', + 'csisolatin5': 'windows-1254', + 'csisolatin6': 'iso8859-10', + 'csisolatinarabic': 'iso8859-6', + 'csisolatincyrillic': 'iso8859-5', + 'csisolatingreek': 'iso8859-7', + 'csisolatinhebrew': 'iso8859-8', + 'cskoi8r': 'koi8-r', + 'csksc56011987': 'cp949', + 'cspc775baltic': 'cp775', + 'cspc850multilingual': 'cp850', + 'cspc862latinhebrew': 'cp862', + 'cspc8codepage437': 'cp437', + 'cspcp852': 'cp852', + 'csptcp154': 'ptcp154', + 'csshiftjis': 'shift_jis', + 'csunicode11utf7': 'utf-7', + 'cyrillic': 'iso8859-5', + 'cyrillicasian': 'ptcp154', + 'ebcdiccpbe': 'cp500', + 'ebcdiccpca': 'cp037', + 'ebcdiccpch': 'cp500', + 'ebcdiccphe': 'cp424', + 'ebcdiccpnl': 'cp037', + 'ebcdiccpus': 'cp037', + 'ebcdiccpwt': 'cp037', + 'ecma114': 'iso8859-6', + 'ecma118': 'iso8859-7', + 'elot928': 'iso8859-7', + 'eucjp': 'euc_jp', + 'euckr': 'cp949', + 'extendedunixcodepackedformatforjapanese': 'euc_jp', + 'gb18030': 'gb18030', + 'gb2312': 'gbk', + 'gb231280': 'gbk', + 'gbk': 'gbk', + 'greek': 'iso8859-7', + 'greek8': 'iso8859-7', + 'hebrew': 'iso8859-8', + 'hproman8': 'hp-roman8', + 'hzgb2312': 'hz', + 'ibm037': 'cp037', + 'ibm1026': 'cp1026', + 'ibm367': 'ascii', + 'ibm424': 'cp424', + 'ibm437': 'cp437', + 'ibm500': 'cp500', + 'ibm775': 'cp775', + 'ibm819': 'windows-1252', + 'ibm850': 'cp850', + 'ibm852': 'cp852', + 'ibm855': 'cp855', + 'ibm857': 'cp857', + 'ibm860': 'cp860', + 'ibm861': 'cp861', + 'ibm862': 'cp862', + 'ibm863': 'cp863', + 'ibm864': 'cp864', + 'ibm865': 'cp865', + 'ibm866': 'cp866', + 'ibm869': 'cp869', + 'iso2022jp': 'iso2022_jp', + 'iso2022jp2': 'iso2022_jp_2', + 'iso2022kr': 'iso2022_kr', + 'iso646irv1991': 'ascii', + 'iso646us': 'ascii', + 'iso88591': 'windows-1252', + 'iso885910': 'iso8859-10', + 'iso8859101992': 'iso8859-10', + 'iso885911987': 'windows-1252', + 'iso885913': 'iso8859-13', + 'iso885914': 'iso8859-14', + 'iso8859141998': 'iso8859-14', + 'iso885915': 'iso8859-15', + 'iso885916': 'iso8859-16', + 'iso8859162001': 'iso8859-16', + 'iso88592': 'iso8859-2', + 'iso885921987': 'iso8859-2', + 'iso88593': 'iso8859-3', + 'iso885931988': 'iso8859-3', + 'iso88594': 'iso8859-4', + 'iso885941988': 'iso8859-4', + 'iso88595': 'iso8859-5', + 'iso885951988': 'iso8859-5', + 'iso88596': 'iso8859-6', + 'iso885961987': 'iso8859-6', + 'iso88597': 'iso8859-7', + 'iso885971987': 'iso8859-7', + 'iso88598': 'iso8859-8', + 'iso885981988': 'iso8859-8', + 'iso88599': 'windows-1254', + 'iso885991989': 'windows-1254', + 'isoceltic': 'iso8859-14', + 'isoir100': 'windows-1252', + 'isoir101': 'iso8859-2', + 'isoir109': 'iso8859-3', + 'isoir110': 'iso8859-4', + 'isoir126': 'iso8859-7', + 'isoir127': 'iso8859-6', + 'isoir138': 'iso8859-8', + 'isoir144': 'iso8859-5', + 'isoir148': 'windows-1254', + 'isoir149': 'cp949', + 'isoir157': 'iso8859-10', + 'isoir199': 'iso8859-14', + 'isoir226': 'iso8859-16', + 'isoir58': 'gbk', + 'isoir6': 'ascii', + 'koi8r': 'koi8-r', + 'koi8u': 'koi8-u', + 'korean': 'cp949', + 'ksc5601': 'cp949', + 'ksc56011987': 'cp949', + 'ksc56011989': 'cp949', + 'l1': 'windows-1252', + 'l10': 'iso8859-16', + 'l2': 'iso8859-2', + 'l3': 'iso8859-3', + 'l4': 'iso8859-4', + 'l5': 'windows-1254', + 'l6': 'iso8859-10', + 'l8': 'iso8859-14', + 'latin1': 'windows-1252', + 'latin10': 'iso8859-16', + 'latin2': 'iso8859-2', + 'latin3': 'iso8859-3', + 'latin4': 'iso8859-4', + 'latin5': 'windows-1254', + 'latin6': 'iso8859-10', + 'latin8': 'iso8859-14', + 'latin9': 'iso8859-15', + 'ms936': 'gbk', + 'mskanji': 'shift_jis', + 'pt154': 'ptcp154', + 'ptcp154': 'ptcp154', + 'r8': 'hp-roman8', + 'roman8': 'hp-roman8', + 'shiftjis': 'shift_jis', + 'tis620': 'cp874', + 'unicode11utf7': 'utf-7', + 'us': 'ascii', + 'usascii': 'ascii', + 'utf16': 'utf-16', + 'utf16be': 'utf-16-be', + 'utf16le': 'utf-16-le', + 'utf8': 'utf-8', + 'windows1250': 'cp1250', + 'windows1251': 'cp1251', + 'windows1252': 'cp1252', + 'windows1253': 'cp1253', + 'windows1254': 'cp1254', + 'windows1255': 'cp1255', + 'windows1256': 'cp1256', + 'windows1257': 'cp1257', + 'windows1258': 'cp1258', + 'windows936': 'gbk', + 'x-x-big5': 'big5'} + +tokenTypes = { + "Doctype": 0, + "Characters": 1, + "SpaceCharacters": 2, + "StartTag": 3, + "EndTag": 4, + "EmptyTag": 5, + "Comment": 6, + "ParseError": 7 +} + +tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"], + tokenTypes["EmptyTag"])) + + +prefixes = dict([(v, k) for k, v in namespaces.items()]) +prefixes["http://www.w3.org/1998/Math/MathML"] = "math" + + +class DataLossWarning(UserWarning): + pass + + +class ReparseException(Exception): + pass diff --git a/html5lib/filters/__init__.py b/html5lib/filters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/html5lib/filters/_base.py b/html5lib/filters/_base.py new file mode 100644 index 0000000..c7dbaed --- /dev/null +++ b/html5lib/filters/_base.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + + +class Filter(object): + def __init__(self, source): + self.source = source + + def __iter__(self): + return iter(self.source) + + def __getattr__(self, name): + return getattr(self.source, name) diff --git a/html5lib/filters/alphabeticalattributes.py b/html5lib/filters/alphabeticalattributes.py new file mode 100644 index 0000000..fed6996 --- /dev/null +++ b/html5lib/filters/alphabeticalattributes.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, division, unicode_literals + +from . import _base + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + + +class Filter(_base.Filter): + def __iter__(self): + for token in _base.Filter.__iter__(self): + if token["type"] in ("StartTag", "EmptyTag"): + attrs = OrderedDict() + for name, value in sorted(token["data"].items(), + key=lambda x: x[0]): + attrs[name] = value + token["data"] = attrs + yield token diff --git a/html5lib/filters/inject_meta_charset.py b/html5lib/filters/inject_meta_charset.py new file mode 100644 index 0000000..ca33b70 --- /dev/null +++ b/html5lib/filters/inject_meta_charset.py @@ -0,0 +1,65 @@ +from __future__ import absolute_import, division, unicode_literals + +from . import _base + + +class Filter(_base.Filter): + def __init__(self, source, encoding): + _base.Filter.__init__(self, source) + self.encoding = encoding + + def __iter__(self): + state = "pre_head" + meta_found = (self.encoding is None) + pending = [] + + for token in _base.Filter.__iter__(self): + type = token["type"] + if type == "StartTag": + if token["name"].lower() == "head": + state = "in_head" + + elif type == "EmptyTag": + if token["name"].lower() == "meta": + # replace charset with actual encoding + has_http_equiv_content_type = False + for (namespace, name), value in token["data"].items(): + if namespace is not None: + continue + elif name.lower() == 'charset': + token["data"][(namespace, name)] = self.encoding + meta_found = True + break + elif name == 'http-equiv' and value.lower() == 'content-type': + has_http_equiv_content_type = True + else: + if has_http_equiv_content_type and (None, "content") in token["data"]: + token["data"][(None, "content")] = 'text/html; charset=%s' % self.encoding + meta_found = True + + elif token["name"].lower() == "head" and not meta_found: + # insert meta into empty head + yield {"type": "StartTag", "name": "head", + "data": token["data"]} + yield {"type": "EmptyTag", "name": "meta", + "data": {(None, "charset"): self.encoding}} + yield {"type": "EndTag", "name": "head"} + meta_found = True + continue + + elif type == "EndTag": + if token["name"].lower() == "head" and pending: + # insert meta into head (if necessary) and flush pending queue + yield pending.pop(0) + if not meta_found: + yield {"type": "EmptyTag", "name": "meta", + "data": {(None, "charset"): self.encoding}} + while pending: + yield pending.pop(0) + meta_found = True + state = "post_head" + + if state == "in_head": + pending.append(token) + else: + yield token diff --git a/html5lib/filters/lint.py b/html5lib/filters/lint.py new file mode 100644 index 0000000..7cc99a4 --- /dev/null +++ b/html5lib/filters/lint.py @@ -0,0 +1,93 @@ +from __future__ import absolute_import, division, unicode_literals + +from gettext import gettext +_ = gettext + +from . import _base +from ..constants import cdataElements, rcdataElements, voidElements + +from ..constants import spaceCharacters +spaceCharacters = "".join(spaceCharacters) + + +class LintError(Exception): + pass + + +class Filter(_base.Filter): + def __iter__(self): + open_elements = [] + contentModelFlag = "PCDATA" + for token in _base.Filter.__iter__(self): + type = token["type"] + if type in ("StartTag", "EmptyTag"): + name = token["name"] + if contentModelFlag != "PCDATA": + raise LintError(_("StartTag not in PCDATA content model flag: %(tag)s") % {"tag": name}) + if not isinstance(name, str): + raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name}) + if not name: + raise LintError(_("Empty tag name")) + if type == "StartTag" and name in voidElements: + raise LintError(_("Void element reported as StartTag token: %(tag)s") % {"tag": name}) + elif type == "EmptyTag" and name not in voidElements: + raise LintError(_("Non-void element reported as EmptyTag token: %(tag)s") % {"tag": token["name"]}) + if type == "StartTag": + open_elements.append(name) + for name, value in token["data"]: + if not isinstance(name, str): + raise LintError(_("Attribute name is not a string: %(name)r") % {"name": name}) + if not name: + raise LintError(_("Empty attribute name")) + if not isinstance(value, str): + raise LintError(_("Attribute value is not a string: %(value)r") % {"value": value}) + if name in cdataElements: + contentModelFlag = "CDATA" + elif name in rcdataElements: + contentModelFlag = "RCDATA" + elif name == "plaintext": + contentModelFlag = "PLAINTEXT" + + elif type == "EndTag": + name = token["name"] + if not isinstance(name, str): + raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name}) + if not name: + raise LintError(_("Empty tag name")) + if name in voidElements: + raise LintError(_("Void element reported as EndTag token: %(tag)s") % {"tag": name}) + start_name = open_elements.pop() + if start_name != name: + raise LintError(_("EndTag (%(end)s) does not match StartTag (%(start)s)") % {"end": name, "start": start_name}) + contentModelFlag = "PCDATA" + + elif type == "Comment": + if contentModelFlag != "PCDATA": + raise LintError(_("Comment not in PCDATA content model flag")) + + elif type in ("Characters", "SpaceCharacters"): + data = token["data"] + if not isinstance(data, str): + raise LintError(_("Attribute name is not a string: %(name)r") % {"name": data}) + if not data: + raise LintError(_("%(type)s token with empty data") % {"type": type}) + if type == "SpaceCharacters": + data = data.strip(spaceCharacters) + if data: + raise LintError(_("Non-space character(s) found in SpaceCharacters token: %(token)r") % {"token": data}) + + elif type == "Doctype": + name = token["name"] + if contentModelFlag != "PCDATA": + raise LintError(_("Doctype not in PCDATA content model flag: %(name)s") % {"name": name}) + if not isinstance(name, str): + raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name}) + # XXX: what to do with token["data"] ? + + elif type in ("ParseError", "SerializeError"): + pass + + else: + raise LintError(_("Unknown token type: %(type)s") % {"type": type}) + + yield token diff --git a/html5lib/filters/optionaltags.py b/html5lib/filters/optionaltags.py new file mode 100644 index 0000000..fefe0b3 --- /dev/null +++ b/html5lib/filters/optionaltags.py @@ -0,0 +1,205 @@ +from __future__ import absolute_import, division, unicode_literals + +from . import _base + + +class Filter(_base.Filter): + def slider(self): + previous1 = previous2 = None + for token in self.source: + if previous1 is not None: + yield previous2, previous1, token + previous2 = previous1 + previous1 = token + yield previous2, previous1, None + + def __iter__(self): + for previous, token, next in self.slider(): + type = token["type"] + if type == "StartTag": + if (token["data"] or + not self.is_optional_start(token["name"], previous, next)): + yield token + elif type == "EndTag": + if not self.is_optional_end(token["name"], next): + yield token + else: + yield token + + def is_optional_start(self, tagname, previous, next): + type = next and next["type"] or None + if tagname in 'html': + # An html element's start tag may be omitted if the first thing + # inside the html element is not a space character or a comment. + return type not in ("Comment", "SpaceCharacters") + elif tagname == 'head': + # A head element's start tag may be omitted if the first thing + # inside the head element is an element. + # XXX: we also omit the start tag if the head element is empty + if type in ("StartTag", "EmptyTag"): + return True + elif type == "EndTag": + return next["name"] == "head" + elif tagname == 'body': + # A body element's start tag may be omitted if the first thing + # inside the body element is not a space character or a comment, + # except if the first thing inside the body element is a script + # or style element and the node immediately preceding the body + # element is a head element whose end tag has been omitted. + if type in ("Comment", "SpaceCharacters"): + return False + elif type == "StartTag": + # XXX: we do not look at the preceding event, so we never omit + # the body element's start tag if it's followed by a script or + # a style element. + return next["name"] not in ('script', 'style') + else: + return True + elif tagname == 'colgroup': + # A colgroup element's start tag may be omitted if the first thing + # inside the colgroup element is a col element, and if the element + # is not immediately preceeded by another colgroup element whose + # end tag has been omitted. + if type in ("StartTag", "EmptyTag"): + # XXX: we do not look at the preceding event, so instead we never + # omit the colgroup element's end tag when it is immediately + # followed by another colgroup element. See is_optional_end. + return next["name"] == "col" + else: + return False + elif tagname == 'tbody': + # A tbody element's start tag may be omitted if the first thing + # inside the tbody element is a tr element, and if the element is + # not immediately preceeded by a tbody, thead, or tfoot element + # whose end tag has been omitted. + if type == "StartTag": + # omit the thead and tfoot elements' end tag when they are + # immediately followed by a tbody element. See is_optional_end. + if previous and previous['type'] == 'EndTag' and \ + previous['name'] in ('tbody', 'thead', 'tfoot'): + return False + return next["name"] == 'tr' + else: + return False + return False + + def is_optional_end(self, tagname, next): + type = next and next["type"] or None + if tagname in ('html', 'head', 'body'): + # An html element's end tag may be omitted if the html element + # is not immediately followed by a space character or a comment. + return type not in ("Comment", "SpaceCharacters") + elif tagname in ('li', 'optgroup', 'tr'): + # A li element's end tag may be omitted if the li element is + # immediately followed by another li element or if there is + # no more content in the parent element. + # An optgroup element's end tag may be omitted if the optgroup + # element is immediately followed by another optgroup element, + # or if there is no more content in the parent element. + # A tr element's end tag may be omitted if the tr element is + # immediately followed by another tr element, or if there is + # no more content in the parent element. + if type == "StartTag": + return next["name"] == tagname + else: + return type == "EndTag" or type is None + elif tagname in ('dt', 'dd'): + # A dt element's end tag may be omitted if the dt element is + # immediately followed by another dt element or a dd element. + # A dd element's end tag may be omitted if the dd element is + # immediately followed by another dd element or a dt element, + # or if there is no more content in the parent element. + if type == "StartTag": + return next["name"] in ('dt', 'dd') + elif tagname == 'dd': + return type == "EndTag" or type is None + else: + return False + elif tagname == 'p': + # A p element's end tag may be omitted if the p element is + # immediately followed by an address, article, aside, + # blockquote, datagrid, dialog, dir, div, dl, fieldset, + # footer, form, h1, h2, h3, h4, h5, h6, header, hr, menu, + # nav, ol, p, pre, section, table, or ul, element, or if + # there is no more content in the parent element. + if type in ("StartTag", "EmptyTag"): + return next["name"] in ('address', 'article', 'aside', + 'blockquote', 'datagrid', 'dialog', + 'dir', 'div', 'dl', 'fieldset', 'footer', + 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', + 'header', 'hr', 'menu', 'nav', 'ol', + 'p', 'pre', 'section', 'table', 'ul') + else: + return type == "EndTag" or type is None + elif tagname == 'option': + # An option element's end tag may be omitted if the option + # element is immediately followed by another option element, + # or if it is immediately followed by an optgroup + # element, or if there is no more content in the parent + # element. + if type == "StartTag": + return next["name"] in ('option', 'optgroup') + else: + return type == "EndTag" or type is None + elif tagname in ('rt', 'rp'): + # An rt element's end tag may be omitted if the rt element is + # immediately followed by an rt or rp element, or if there is + # no more content in the parent element. + # An rp element's end tag may be omitted if the rp element is + # immediately followed by an rt or rp element, or if there is + # no more content in the parent element. + if type == "StartTag": + return next["name"] in ('rt', 'rp') + else: + return type == "EndTag" or type is None + elif tagname == 'colgroup': + # A colgroup element's end tag may be omitted if the colgroup + # element is not immediately followed by a space character or + # a comment. + if type in ("Comment", "SpaceCharacters"): + return False + elif type == "StartTag": + # XXX: we also look for an immediately following colgroup + # element. See is_optional_start. + return next["name"] != 'colgroup' + else: + return True + elif tagname in ('thead', 'tbody'): + # A thead element's end tag may be omitted if the thead element + # is immediately followed by a tbody or tfoot element. + # A tbody element's end tag may be omitted if the tbody element + # is immediately followed by a tbody or tfoot element, or if + # there is no more content in the parent element. + # A tfoot element's end tag may be omitted if the tfoot element + # is immediately followed by a tbody element, or if there is no + # more content in the parent element. + # XXX: we never omit the end tag when the following element is + # a tbody. See is_optional_start. + if type == "StartTag": + return next["name"] in ['tbody', 'tfoot'] + elif tagname == 'tbody': + return type == "EndTag" or type is None + else: + return False + elif tagname == 'tfoot': + # A tfoot element's end tag may be omitted if the tfoot element + # is immediately followed by a tbody element, or if there is no + # more content in the parent element. + # XXX: we never omit the end tag when the following element is + # a tbody. See is_optional_start. + if type == "StartTag": + return next["name"] == 'tbody' + else: + return type == "EndTag" or type is None + elif tagname in ('td', 'th'): + # A td element's end tag may be omitted if the td element is + # immediately followed by a td or th element, or if there is + # no more content in the parent element. + # A th element's end tag may be omitted if the th element is + # immediately followed by a td or th element, or if there is + # no more content in the parent element. + if type == "StartTag": + return next["name"] in ('td', 'th') + else: + return type == "EndTag" or type is None + return False diff --git a/html5lib/filters/sanitizer.py b/html5lib/filters/sanitizer.py new file mode 100644 index 0000000..b206b54 --- /dev/null +++ b/html5lib/filters/sanitizer.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + +from . import _base +from ..sanitizer import HTMLSanitizerMixin + + +class Filter(_base.Filter, HTMLSanitizerMixin): + def __iter__(self): + for token in _base.Filter.__iter__(self): + token = self.sanitize_token(token) + if token: + yield token diff --git a/html5lib/filters/whitespace.py b/html5lib/filters/whitespace.py new file mode 100644 index 0000000..dfc60ee --- /dev/null +++ b/html5lib/filters/whitespace.py @@ -0,0 +1,38 @@ +from __future__ import absolute_import, division, unicode_literals + +import re + +from . import _base +from ..constants import rcdataElements, spaceCharacters +spaceCharacters = "".join(spaceCharacters) + +SPACES_REGEX = re.compile("[%s]+" % spaceCharacters) + + +class Filter(_base.Filter): + + spacePreserveElements = frozenset(["pre", "textarea"] + list(rcdataElements)) + + def __iter__(self): + preserve = 0 + for token in _base.Filter.__iter__(self): + type = token["type"] + if type == "StartTag" \ + and (preserve or token["name"] in self.spacePreserveElements): + preserve += 1 + + elif type == "EndTag" and preserve: + preserve -= 1 + + elif not preserve and type == "SpaceCharacters" and token["data"]: + # Test on token["data"] above to not introduce spaces where there were not + token["data"] = " " + + elif not preserve and type == "Characters": + token["data"] = collapse_spaces(token["data"]) + + yield token + + +def collapse_spaces(text): + return SPACES_REGEX.sub(' ', text) diff --git a/html5lib/html5parser.py b/html5lib/html5parser.py new file mode 100644 index 0000000..b0f14f3 --- /dev/null +++ b/html5lib/html5parser.py @@ -0,0 +1,2713 @@ +from __future__ import absolute_import, division, unicode_literals +from six import with_metaclass + +import types + +from . import inputstream +from . import tokenizer + +from . import treebuilders +from .treebuilders._base import Marker + +from . import utils +from . import constants +from .constants import spaceCharacters, asciiUpper2Lower +from .constants import specialElements +from .constants import headingElements +from .constants import cdataElements, rcdataElements +from .constants import tokenTypes, ReparseException, namespaces +from .constants import htmlIntegrationPointElements, mathmlTextIntegrationPointElements +from .constants import adjustForeignAttributes as adjustForeignAttributesMap + + +def parse(doc, treebuilder="etree", encoding=None, + namespaceHTMLElements=True): + """Parse a string or file-like object into a tree""" + tb = treebuilders.getTreeBuilder(treebuilder) + p = HTMLParser(tb, namespaceHTMLElements=namespaceHTMLElements) + return p.parse(doc, encoding=encoding) + + +def parseFragment(doc, container="div", treebuilder="etree", encoding=None, + namespaceHTMLElements=True): + tb = treebuilders.getTreeBuilder(treebuilder) + p = HTMLParser(tb, namespaceHTMLElements=namespaceHTMLElements) + return p.parseFragment(doc, container=container, encoding=encoding) + + +def method_decorator_metaclass(function): + class Decorated(type): + def __new__(meta, classname, bases, classDict): + for attributeName, attribute in classDict.items(): + if isinstance(attribute, types.FunctionType): + attribute = function(attribute) + + classDict[attributeName] = attribute + return type.__new__(meta, classname, bases, classDict) + return Decorated + + +class HTMLParser(object): + """HTML parser. Generates a tree structure from a stream of (possibly + malformed) HTML""" + + def __init__(self, tree=None, tokenizer=tokenizer.HTMLTokenizer, + strict=False, namespaceHTMLElements=True, debug=False): + """ + strict - raise an exception when a parse error is encountered + + tree - a treebuilder class controlling the type of tree that will be + returned. Built in treebuilders can be accessed through + html5lib.treebuilders.getTreeBuilder(treeType) + + tokenizer - a class that provides a stream of tokens to the treebuilder. + This may be replaced for e.g. a sanitizer which converts some tags to + text + """ + + # Raise an exception on the first error encountered + self.strict = strict + + if tree is None: + tree = treebuilders.getTreeBuilder("etree") + self.tree = tree(namespaceHTMLElements) + self.tokenizer_class = tokenizer + self.errors = [] + + self.phases = dict([(name, cls(self, self.tree)) for name, cls in + getPhases(debug).items()]) + + def _parse(self, stream, innerHTML=False, container="div", + encoding=None, parseMeta=True, useChardet=True, **kwargs): + + self.innerHTMLMode = innerHTML + self.container = container + self.tokenizer = self.tokenizer_class(stream, encoding=encoding, + parseMeta=parseMeta, + useChardet=useChardet, + parser=self, **kwargs) + self.reset() + + while True: + try: + self.mainLoop() + break + except ReparseException: + self.reset() + + def reset(self): + self.tree.reset() + self.firstStartTag = False + self.errors = [] + self.log = [] # only used with debug mode + # "quirks" / "limited quirks" / "no quirks" + self.compatMode = "no quirks" + + if self.innerHTMLMode: + self.innerHTML = self.container.lower() + + if self.innerHTML in cdataElements: + self.tokenizer.state = self.tokenizer.rcdataState + elif self.innerHTML in rcdataElements: + self.tokenizer.state = self.tokenizer.rawtextState + elif self.innerHTML == 'plaintext': + self.tokenizer.state = self.tokenizer.plaintextState + else: + # state already is data state + # self.tokenizer.state = self.tokenizer.dataState + pass + self.phase = self.phases["beforeHtml"] + self.phase.insertHtmlElement() + self.resetInsertionMode() + else: + self.innerHTML = False + self.phase = self.phases["initial"] + + self.lastPhase = None + + self.beforeRCDataPhase = None + + self.framesetOK = True + + def isHTMLIntegrationPoint(self, element): + if (element.name == "annotation-xml" and + element.namespace == namespaces["mathml"]): + return ("encoding" in element.attributes and + element.attributes["encoding"].translate( + asciiUpper2Lower) in + ("text/html", "application/xhtml+xml")) + else: + return (element.namespace, element.name) in htmlIntegrationPointElements + + def isMathMLTextIntegrationPoint(self, element): + return (element.namespace, element.name) in mathmlTextIntegrationPointElements + + def mainLoop(self): + CharactersToken = tokenTypes["Characters"] + SpaceCharactersToken = tokenTypes["SpaceCharacters"] + StartTagToken = tokenTypes["StartTag"] + EndTagToken = tokenTypes["EndTag"] + CommentToken = tokenTypes["Comment"] + DoctypeToken = tokenTypes["Doctype"] + ParseErrorToken = tokenTypes["ParseError"] + + for token in self.normalizedTokens(): + new_token = token + while new_token is not None: + currentNode = self.tree.openElements[-1] if self.tree.openElements else None + currentNodeNamespace = currentNode.namespace if currentNode else None + currentNodeName = currentNode.name if currentNode else None + + type = new_token["type"] + + if type == ParseErrorToken: + self.parseError(new_token["data"], new_token.get("datavars", {})) + new_token = None + else: + if (len(self.tree.openElements) == 0 or + currentNodeNamespace == self.tree.defaultNamespace or + (self.isMathMLTextIntegrationPoint(currentNode) and + ((type == StartTagToken and + token["name"] not in frozenset(["mglyph", "malignmark"])) or + type in (CharactersToken, SpaceCharactersToken))) or + (currentNodeNamespace == namespaces["mathml"] and + currentNodeName == "annotation-xml" and + token["name"] == "svg") or + (self.isHTMLIntegrationPoint(currentNode) and + type in (StartTagToken, CharactersToken, SpaceCharactersToken))): + phase = self.phase + else: + phase = self.phases["inForeignContent"] + + if type == CharactersToken: + new_token = phase.processCharacters(new_token) + elif type == SpaceCharactersToken: + new_token = phase.processSpaceCharacters(new_token) + elif type == StartTagToken: + new_token = phase.processStartTag(new_token) + elif type == EndTagToken: + new_token = phase.processEndTag(new_token) + elif type == CommentToken: + new_token = phase.processComment(new_token) + elif type == DoctypeToken: + new_token = phase.processDoctype(new_token) + + if (type == StartTagToken and token["selfClosing"] + and not token["selfClosingAcknowledged"]): + self.parseError("non-void-element-with-trailing-solidus", + {"name": token["name"]}) + + # When the loop finishes it's EOF + reprocess = True + phases = [] + while reprocess: + phases.append(self.phase) + reprocess = self.phase.processEOF() + if reprocess: + assert self.phase not in phases + + def normalizedTokens(self): + for token in self.tokenizer: + yield self.normalizeToken(token) + + def parse(self, stream, encoding=None, parseMeta=True, useChardet=True): + """Parse a HTML document into a well-formed tree + + stream - a filelike object or string containing the HTML to be parsed + + The optional encoding parameter must be a string that indicates + the encoding. If specified, that encoding will be used, + regardless of any BOM or later declaration (such as in a meta + element) + """ + self._parse(stream, innerHTML=False, encoding=encoding, + parseMeta=parseMeta, useChardet=useChardet) + return self.tree.getDocument() + + def parseFragment(self, stream, container="div", encoding=None, + parseMeta=False, useChardet=True): + """Parse a HTML fragment into a well-formed tree fragment + + container - name of the element we're setting the innerHTML property + if set to None, default to 'div' + + stream - a filelike object or string containing the HTML to be parsed + + The optional encoding parameter must be a string that indicates + the encoding. If specified, that encoding will be used, + regardless of any BOM or later declaration (such as in a meta + element) + """ + self._parse(stream, True, container=container, encoding=encoding) + return self.tree.getFragment() + + def parseError(self, errorcode="XXX-undefined-error", datavars={}): + # XXX The idea is to make errorcode mandatory. + self.errors.append((self.tokenizer.stream.position(), errorcode, datavars)) + if self.strict: + raise ParseError + + def normalizeToken(self, token): + """ HTML5 specific normalizations to the token stream """ + + if token["type"] == tokenTypes["StartTag"]: + token["data"] = dict(token["data"][::-1]) + + return token + + def adjustMathMLAttributes(self, token): + replacements = {"definitionurl": "definitionURL"} + for k, v in replacements.items(): + if k in token["data"]: + token["data"][v] = token["data"][k] + del token["data"][k] + + def adjustSVGAttributes(self, token): + replacements = { + "attributename": "attributeName", + "attributetype": "attributeType", + "basefrequency": "baseFrequency", + "baseprofile": "baseProfile", + "calcmode": "calcMode", + "clippathunits": "clipPathUnits", + "contentscripttype": "contentScriptType", + "contentstyletype": "contentStyleType", + "diffuseconstant": "diffuseConstant", + "edgemode": "edgeMode", + "externalresourcesrequired": "externalResourcesRequired", + "filterres": "filterRes", + "filterunits": "filterUnits", + "glyphref": "glyphRef", + "gradienttransform": "gradientTransform", + "gradientunits": "gradientUnits", + "kernelmatrix": "kernelMatrix", + "kernelunitlength": "kernelUnitLength", + "keypoints": "keyPoints", + "keysplines": "keySplines", + "keytimes": "keyTimes", + "lengthadjust": "lengthAdjust", + "limitingconeangle": "limitingConeAngle", + "markerheight": "markerHeight", + "markerunits": "markerUnits", + "markerwidth": "markerWidth", + "maskcontentunits": "maskContentUnits", + "maskunits": "maskUnits", + "numoctaves": "numOctaves", + "pathlength": "pathLength", + "patterncontentunits": "patternContentUnits", + "patterntransform": "patternTransform", + "patternunits": "patternUnits", + "pointsatx": "pointsAtX", + "pointsaty": "pointsAtY", + "pointsatz": "pointsAtZ", + "preservealpha": "preserveAlpha", + "preserveaspectratio": "preserveAspectRatio", + "primitiveunits": "primitiveUnits", + "refx": "refX", + "refy": "refY", + "repeatcount": "repeatCount", + "repeatdur": "repeatDur", + "requiredextensions": "requiredExtensions", + "requiredfeatures": "requiredFeatures", + "specularconstant": "specularConstant", + "specularexponent": "specularExponent", + "spreadmethod": "spreadMethod", + "startoffset": "startOffset", + "stddeviation": "stdDeviation", + "stitchtiles": "stitchTiles", + "surfacescale": "surfaceScale", + "systemlanguage": "systemLanguage", + "tablevalues": "tableValues", + "targetx": "targetX", + "targety": "targetY", + "textlength": "textLength", + "viewbox": "viewBox", + "viewtarget": "viewTarget", + "xchannelselector": "xChannelSelector", + "ychannelselector": "yChannelSelector", + "zoomandpan": "zoomAndPan" + } + for originalName in list(token["data"].keys()): + if originalName in replacements: + svgName = replacements[originalName] + token["data"][svgName] = token["data"][originalName] + del token["data"][originalName] + + def adjustForeignAttributes(self, token): + replacements = adjustForeignAttributesMap + + for originalName in token["data"].keys(): + if originalName in replacements: + foreignName = replacements[originalName] + token["data"][foreignName] = token["data"][originalName] + del token["data"][originalName] + + def reparseTokenNormal(self, token): + self.parser.phase() + + def resetInsertionMode(self): + # The name of this method is mostly historical. (It's also used in the + # specification.) + last = False + newModes = { + "select": "inSelect", + "td": "inCell", + "th": "inCell", + "tr": "inRow", + "tbody": "inTableBody", + "thead": "inTableBody", + "tfoot": "inTableBody", + "caption": "inCaption", + "colgroup": "inColumnGroup", + "table": "inTable", + "head": "inBody", + "body": "inBody", + "frameset": "inFrameset", + "html": "beforeHead" + } + for node in self.tree.openElements[::-1]: + nodeName = node.name + new_phase = None + if node == self.tree.openElements[0]: + assert self.innerHTML + last = True + nodeName = self.innerHTML + # Check for conditions that should only happen in the innerHTML + # case + if nodeName in ("select", "colgroup", "head", "html"): + assert self.innerHTML + + if not last and node.namespace != self.tree.defaultNamespace: + continue + + if nodeName in newModes: + new_phase = self.phases[newModes[nodeName]] + break + elif last: + new_phase = self.phases["inBody"] + break + + self.phase = new_phase + + def parseRCDataRawtext(self, token, contentType): + """Generic RCDATA/RAWTEXT Parsing algorithm + contentType - RCDATA or RAWTEXT + """ + assert contentType in ("RAWTEXT", "RCDATA") + + self.tree.insertElement(token) + + if contentType == "RAWTEXT": + self.tokenizer.state = self.tokenizer.rawtextState + else: + self.tokenizer.state = self.tokenizer.rcdataState + + self.originalPhase = self.phase + + self.phase = self.phases["text"] + + +def getPhases(debug): + def log(function): + """Logger that records which phase processes each token""" + type_names = dict((value, key) for key, value in + constants.tokenTypes.items()) + + def wrapped(self, *args, **kwargs): + if function.__name__.startswith("process") and len(args) > 0: + token = args[0] + try: + info = {"type": type_names[token['type']]} + except: + raise + if token['type'] in constants.tagTokenTypes: + info["name"] = token['name'] + + self.parser.log.append((self.parser.tokenizer.state.__name__, + self.parser.phase.__class__.__name__, + self.__class__.__name__, + function.__name__, + info)) + return function(self, *args, **kwargs) + else: + return function(self, *args, **kwargs) + return wrapped + + def getMetaclass(use_metaclass, metaclass_func): + if use_metaclass: + return method_decorator_metaclass(metaclass_func) + else: + return type + + class Phase(with_metaclass(getMetaclass(debug, log))): + """Base class for helper object that implements each phase of processing + """ + + def __init__(self, parser, tree): + self.parser = parser + self.tree = tree + + def processEOF(self): + raise NotImplementedError + + def processComment(self, token): + # For most phases the following is correct. Where it's not it will be + # overridden. + self.tree.insertComment(token, self.tree.openElements[-1]) + + def processDoctype(self, token): + self.parser.parseError("unexpected-doctype") + + def processCharacters(self, token): + self.tree.insertText(token["data"]) + + def processSpaceCharacters(self, token): + self.tree.insertText(token["data"]) + + def processStartTag(self, token): + return self.startTagHandler[token["name"]](token) + + def startTagHtml(self, token): + if not self.parser.firstStartTag and token["name"] == "html": + self.parser.parseError("non-html-root") + # XXX Need a check here to see if the first start tag token emitted is + # this token... If it's not, invoke self.parser.parseError(). + for attr, value in token["data"].items(): + if attr not in self.tree.openElements[0].attributes: + self.tree.openElements[0].attributes[attr] = value + self.parser.firstStartTag = False + + def processEndTag(self, token): + return self.endTagHandler[token["name"]](token) + + class InitialPhase(Phase): + def processSpaceCharacters(self, token): + pass + + def processComment(self, token): + self.tree.insertComment(token, self.tree.document) + + def processDoctype(self, token): + name = token["name"] + publicId = token["publicId"] + systemId = token["systemId"] + correct = token["correct"] + + if (name != "html" or publicId is not None or + systemId is not None and systemId != "about:legacy-compat"): + self.parser.parseError("unknown-doctype") + + if publicId is None: + publicId = "" + + self.tree.insertDoctype(token) + + if publicId != "": + publicId = publicId.translate(asciiUpper2Lower) + + if (not correct or token["name"] != "html" + or publicId.startswith( + ("+//silmaril//dtd html pro v0r11 19970101//", + "-//advasoft ltd//dtd html 3.0 aswedit + extensions//", + "-//as//dtd html 3.0 aswedit + extensions//", + "-//ietf//dtd html 2.0 level 1//", + "-//ietf//dtd html 2.0 level 2//", + "-//ietf//dtd html 2.0 strict level 1//", + "-//ietf//dtd html 2.0 strict level 2//", + "-//ietf//dtd html 2.0 strict//", + "-//ietf//dtd html 2.0//", + "-//ietf//dtd html 2.1e//", + "-//ietf//dtd html 3.0//", + "-//ietf//dtd html 3.2 final//", + "-//ietf//dtd html 3.2//", + "-//ietf//dtd html 3//", + "-//ietf//dtd html level 0//", + "-//ietf//dtd html level 1//", + "-//ietf//dtd html level 2//", + "-//ietf//dtd html level 3//", + "-//ietf//dtd html strict level 0//", + "-//ietf//dtd html strict level 1//", + "-//ietf//dtd html strict level 2//", + "-//ietf//dtd html strict level 3//", + "-//ietf//dtd html strict//", + "-//ietf//dtd html//", + "-//metrius//dtd metrius presentational//", + "-//microsoft//dtd internet explorer 2.0 html strict//", + "-//microsoft//dtd internet explorer 2.0 html//", + "-//microsoft//dtd internet explorer 2.0 tables//", + "-//microsoft//dtd internet explorer 3.0 html strict//", + "-//microsoft//dtd internet explorer 3.0 html//", + "-//microsoft//dtd internet explorer 3.0 tables//", + "-//netscape comm. corp.//dtd html//", + "-//netscape comm. corp.//dtd strict html//", + "-//o'reilly and associates//dtd html 2.0//", + "-//o'reilly and associates//dtd html extended 1.0//", + "-//o'reilly and associates//dtd html extended relaxed 1.0//", + "-//softquad software//dtd hotmetal pro 6.0::19990601::extensions to html 4.0//", + "-//softquad//dtd hotmetal pro 4.0::19971010::extensions to html 4.0//", + "-//spyglass//dtd html 2.0 extended//", + "-//sq//dtd html 2.0 hotmetal + extensions//", + "-//sun microsystems corp.//dtd hotjava html//", + "-//sun microsystems corp.//dtd hotjava strict html//", + "-//w3c//dtd html 3 1995-03-24//", + "-//w3c//dtd html 3.2 draft//", + "-//w3c//dtd html 3.2 final//", + "-//w3c//dtd html 3.2//", + "-//w3c//dtd html 3.2s draft//", + "-//w3c//dtd html 4.0 frameset//", + "-//w3c//dtd html 4.0 transitional//", + "-//w3c//dtd html experimental 19960712//", + "-//w3c//dtd html experimental 970421//", + "-//w3c//dtd w3 html//", + "-//w3o//dtd w3 html 3.0//", + "-//webtechs//dtd mozilla html 2.0//", + "-//webtechs//dtd mozilla html//")) + or publicId in + ("-//w3o//dtd w3 html strict 3.0//en//", + "-/w3c/dtd html 4.0 transitional/en", + "html") + or publicId.startswith( + ("-//w3c//dtd html 4.01 frameset//", + "-//w3c//dtd html 4.01 transitional//")) and + systemId is None + or systemId and systemId.lower() == "http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd"): + self.parser.compatMode = "quirks" + elif (publicId.startswith( + ("-//w3c//dtd xhtml 1.0 frameset//", + "-//w3c//dtd xhtml 1.0 transitional//")) + or publicId.startswith( + ("-//w3c//dtd html 4.01 frameset//", + "-//w3c//dtd html 4.01 transitional//")) and + systemId is not None): + self.parser.compatMode = "limited quirks" + + self.parser.phase = self.parser.phases["beforeHtml"] + + def anythingElse(self): + self.parser.compatMode = "quirks" + self.parser.phase = self.parser.phases["beforeHtml"] + + def processCharacters(self, token): + self.parser.parseError("expected-doctype-but-got-chars") + self.anythingElse() + return token + + def processStartTag(self, token): + self.parser.parseError("expected-doctype-but-got-start-tag", + {"name": token["name"]}) + self.anythingElse() + return token + + def processEndTag(self, token): + self.parser.parseError("expected-doctype-but-got-end-tag", + {"name": token["name"]}) + self.anythingElse() + return token + + def processEOF(self): + self.parser.parseError("expected-doctype-but-got-eof") + self.anythingElse() + return True + + class BeforeHtmlPhase(Phase): + # helper methods + def insertHtmlElement(self): + self.tree.insertRoot(impliedTagToken("html", "StartTag")) + self.parser.phase = self.parser.phases["beforeHead"] + + # other + def processEOF(self): + self.insertHtmlElement() + return True + + def processComment(self, token): + self.tree.insertComment(token, self.tree.document) + + def processSpaceCharacters(self, token): + pass + + def processCharacters(self, token): + self.insertHtmlElement() + return token + + def processStartTag(self, token): + if token["name"] == "html": + self.parser.firstStartTag = True + self.insertHtmlElement() + return token + + def processEndTag(self, token): + if token["name"] not in ("head", "body", "html", "br"): + self.parser.parseError("unexpected-end-tag-before-html", + {"name": token["name"]}) + else: + self.insertHtmlElement() + return token + + class BeforeHeadPhase(Phase): + def __init__(self, parser, tree): + Phase.__init__(self, parser, tree) + + self.startTagHandler = utils.MethodDispatcher([ + ("html", self.startTagHtml), + ("head", self.startTagHead) + ]) + self.startTagHandler.default = self.startTagOther + + self.endTagHandler = utils.MethodDispatcher([ + (("head", "body", "html", "br"), self.endTagImplyHead) + ]) + self.endTagHandler.default = self.endTagOther + + def processEOF(self): + self.startTagHead(impliedTagToken("head", "StartTag")) + return True + + def processSpaceCharacters(self, token): + pass + + def processCharacters(self, token): + self.startTagHead(impliedTagToken("head", "StartTag")) + return token + + def startTagHtml(self, token): + return self.parser.phases["inBody"].processStartTag(token) + + def startTagHead(self, token): + self.tree.insertElement(token) + self.tree.headPointer = self.tree.openElements[-1] + self.parser.phase = self.parser.phases["inHead"] + + def startTagOther(self, token): + self.startTagHead(impliedTagToken("head", "StartTag")) + return token + + def endTagImplyHead(self, token): + self.startTagHead(impliedTagToken("head", "StartTag")) + return token + + def endTagOther(self, token): + self.parser.parseError("end-tag-after-implied-root", + {"name": token["name"]}) + + class InHeadPhase(Phase): + def __init__(self, parser, tree): + Phase.__init__(self, parser, tree) + + self.startTagHandler = utils.MethodDispatcher([ + ("html", self.startTagHtml), + ("title", self.startTagTitle), + (("noscript", "noframes", "style"), self.startTagNoScriptNoFramesStyle), + ("script", self.startTagScript), + (("base", "basefont", "bgsound", "command", "link"), + self.startTagBaseLinkCommand), + ("meta", self.startTagMeta), + ("head", self.startTagHead) + ]) + self.startTagHandler.default = self.startTagOther + + self. endTagHandler = utils.MethodDispatcher([ + ("head", self.endTagHead), + (("br", "html", "body"), self.endTagHtmlBodyBr) + ]) + self.endTagHandler.default = self.endTagOther + + # the real thing + def processEOF(self): + self.anythingElse() + return True + + def processCharacters(self, token): + self.anythingElse() + return token + + def startTagHtml(self, token): + return self.parser.phases["inBody"].processStartTag(token) + + def startTagHead(self, token): + self.parser.parseError("two-heads-are-not-better-than-one") + + def startTagBaseLinkCommand(self, token): + self.tree.insertElement(token) + self.tree.openElements.pop() + token["selfClosingAcknowledged"] = True + + def startTagMeta(self, token): + self.tree.insertElement(token) + self.tree.openElements.pop() + token["selfClosingAcknowledged"] = True + + attributes = token["data"] + if self.parser.tokenizer.stream.charEncoding[1] == "tentative": + if "charset" in attributes: + self.parser.tokenizer.stream.changeEncoding(attributes["charset"]) + elif ("content" in attributes and + "http-equiv" in attributes and + attributes["http-equiv"].lower() == "content-type"): + # Encoding it as UTF-8 here is a hack, as really we should pass + # the abstract Unicode string, and just use the + # ContentAttrParser on that, but using UTF-8 allows all chars + # to be encoded and as a ASCII-superset works. + data = inputstream.EncodingBytes(attributes["content"].encode("utf-8")) + parser = inputstream.ContentAttrParser(data) + codec = parser.parse() + self.parser.tokenizer.stream.changeEncoding(codec) + + def startTagTitle(self, token): + self.parser.parseRCDataRawtext(token, "RCDATA") + + def startTagNoScriptNoFramesStyle(self, token): + # Need to decide whether to implement the scripting-disabled case + self.parser.parseRCDataRawtext(token, "RAWTEXT") + + def startTagScript(self, token): + self.tree.insertElement(token) + self.parser.tokenizer.state = self.parser.tokenizer.scriptDataState + self.parser.originalPhase = self.parser.phase + self.parser.phase = self.parser.phases["text"] + + def startTagOther(self, token): + self.anythingElse() + return token + + def endTagHead(self, token): + node = self.parser.tree.openElements.pop() + assert node.name == "head", "Expected head got %s" % node.name + self.parser.phase = self.parser.phases["afterHead"] + + def endTagHtmlBodyBr(self, token): + self.anythingElse() + return token + + def endTagOther(self, token): + self.parser.parseError("unexpected-end-tag", {"name": token["name"]}) + + def anythingElse(self): + self.endTagHead(impliedTagToken("head")) + + # XXX If we implement a parser for which scripting is disabled we need to + # implement this phase. + # + # class InHeadNoScriptPhase(Phase): + class AfterHeadPhase(Phase): + def __init__(self, parser, tree): + Phase.__init__(self, parser, tree) + + self.startTagHandler = utils.MethodDispatcher([ + ("html", self.startTagHtml), + ("body", self.startTagBody), + ("frameset", self.startTagFrameset), + (("base", "basefont", "bgsound", "link", "meta", "noframes", "script", + "style", "title"), + self.startTagFromHead), + ("head", self.startTagHead) + ]) + self.startTagHandler.default = self.startTagOther + self.endTagHandler = utils.MethodDispatcher([(("body", "html", "br"), + self.endTagHtmlBodyBr)]) + self.endTagHandler.default = self.endTagOther + + def processEOF(self): + self.anythingElse() + return True + + def processCharacters(self, token): + self.anythingElse() + return token + + def startTagHtml(self, token): + return self.parser.phases["inBody"].processStartTag(token) + + def startTagBody(self, token): + self.parser.framesetOK = False + self.tree.insertElement(token) + self.parser.phase = self.parser.phases["inBody"] + + def startTagFrameset(self, token): + self.tree.insertElement(token) + self.parser.phase = self.parser.phases["inFrameset"] + + def startTagFromHead(self, token): + self.parser.parseError("unexpected-start-tag-out-of-my-head", + {"name": token["name"]}) + self.tree.openElements.append(self.tree.headPointer) + self.parser.phases["inHead"].processStartTag(token) + for node in self.tree.openElements[::-1]: + if node.name == "head": + self.tree.openElements.remove(node) + break + + def startTagHead(self, token): + self.parser.parseError("unexpected-start-tag", {"name": token["name"]}) + + def startTagOther(self, token): + self.anythingElse() + return token + + def endTagHtmlBodyBr(self, token): + self.anythingElse() + return token + + def endTagOther(self, token): + self.parser.parseError("unexpected-end-tag", {"name": token["name"]}) + + def anythingElse(self): + self.tree.insertElement(impliedTagToken("body", "StartTag")) + self.parser.phase = self.parser.phases["inBody"] + self.parser.framesetOK = True + + class InBodyPhase(Phase): + # http://www.whatwg.org/specs/web-apps/current-work/#parsing-main-inbody + # the really-really-really-very crazy mode + def __init__(self, parser, tree): + Phase.__init__(self, parser, tree) + + # Keep a ref to this for special handling of whitespace in 
    +            self.processSpaceCharactersNonPre = self.processSpaceCharacters
+
+            self.startTagHandler = utils.MethodDispatcher([
+                ("html", self.startTagHtml),
+                (("base", "basefont", "bgsound", "command", "link", "meta",
+                  "noframes", "script", "style", "title"),
+                 self.startTagProcessInHead),
+                ("body", self.startTagBody),
+                ("frameset", self.startTagFrameset),
+                (("address", "article", "aside", "blockquote", "center", "details",
+                  "details", "dir", "div", "dl", "fieldset", "figcaption", "figure",
+                  "footer", "header", "hgroup", "main", "menu", "nav", "ol", "p",
+                  "section", "summary", "ul"),
+                 self.startTagCloseP),
+                (headingElements, self.startTagHeading),
+                (("pre", "listing"), self.startTagPreListing),
+                ("form", self.startTagForm),
+                (("li", "dd", "dt"), self.startTagListItem),
+                ("plaintext", self.startTagPlaintext),
+                ("a", self.startTagA),
+                (("b", "big", "code", "em", "font", "i", "s", "small", "strike",
+                  "strong", "tt", "u"), self.startTagFormatting),
+                ("nobr", self.startTagNobr),
+                ("button", self.startTagButton),
+                (("applet", "marquee", "object"), self.startTagAppletMarqueeObject),
+                ("xmp", self.startTagXmp),
+                ("table", self.startTagTable),
+                (("area", "br", "embed", "img", "keygen", "wbr"),
+                 self.startTagVoidFormatting),
+                (("param", "source", "track"), self.startTagParamSource),
+                ("input", self.startTagInput),
+                ("hr", self.startTagHr),
+                ("image", self.startTagImage),
+                ("isindex", self.startTagIsIndex),
+                ("textarea", self.startTagTextarea),
+                ("iframe", self.startTagIFrame),
+                (("noembed", "noframes", "noscript"), self.startTagRawtext),
+                ("select", self.startTagSelect),
+                (("rp", "rt"), self.startTagRpRt),
+                (("option", "optgroup"), self.startTagOpt),
+                (("math"), self.startTagMath),
+                (("svg"), self.startTagSvg),
+                (("caption", "col", "colgroup", "frame", "head",
+                  "tbody", "td", "tfoot", "th", "thead",
+                  "tr"), self.startTagMisplaced)
+            ])
+            self.startTagHandler.default = self.startTagOther
+
+            self.endTagHandler = utils.MethodDispatcher([
+                ("body", self.endTagBody),
+                ("html", self.endTagHtml),
+                (("address", "article", "aside", "blockquote", "button", "center",
+                  "details", "dialog", "dir", "div", "dl", "fieldset", "figcaption", "figure",
+                  "footer", "header", "hgroup", "listing", "main", "menu", "nav", "ol", "pre",
+                  "section", "summary", "ul"), self.endTagBlock),
+                ("form", self.endTagForm),
+                ("p", self.endTagP),
+                (("dd", "dt", "li"), self.endTagListItem),
+                (headingElements, self.endTagHeading),
+                (("a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small",
+                  "strike", "strong", "tt", "u"), self.endTagFormatting),
+                (("applet", "marquee", "object"), self.endTagAppletMarqueeObject),
+                ("br", self.endTagBr),
+            ])
+            self.endTagHandler.default = self.endTagOther
+
+        def isMatchingFormattingElement(self, node1, node2):
+            if node1.name != node2.name or node1.namespace != node2.namespace:
+                return False
+            elif len(node1.attributes) != len(node2.attributes):
+                return False
+            else:
+                attributes1 = sorted(node1.attributes.items())
+                attributes2 = sorted(node2.attributes.items())
+                for attr1, attr2 in zip(attributes1, attributes2):
+                    if attr1 != attr2:
+                        return False
+            return True
+
+        # helper
+        def addFormattingElement(self, token):
+            self.tree.insertElement(token)
+            element = self.tree.openElements[-1]
+
+            matchingElements = []
+            for node in self.tree.activeFormattingElements[::-1]:
+                if node is Marker:
+                    break
+                elif self.isMatchingFormattingElement(node, element):
+                    matchingElements.append(node)
+
+            assert len(matchingElements) <= 3
+            if len(matchingElements) == 3:
+                self.tree.activeFormattingElements.remove(matchingElements[-1])
+            self.tree.activeFormattingElements.append(element)
+
+        # the real deal
+        def processEOF(self):
+            allowed_elements = frozenset(("dd", "dt", "li", "p", "tbody", "td",
+                                          "tfoot", "th", "thead", "tr", "body",
+                                          "html"))
+            for node in self.tree.openElements[::-1]:
+                if node.name not in allowed_elements:
+                    self.parser.parseError("expected-closing-tag-but-got-eof")
+                    break
+            # Stop parsing
+
+        def processSpaceCharactersDropNewline(self, token):
+            # Sometimes (start of 
    , , and