Remove Survey Link from Web Service index.html

The cause is that the function shields breaks inside existing paragraps
  these are seen as preformatted text blocks. Sadly AO3 encapsulates story
  bodies into a p tag...

.gitignore

@@ -0,0 +1,16 @@
+#############
+## Python
+#############
+*.py[cod]
+
+# Translations
+*.mo
+
+# Emacs autosave files
+\#*#
+*~
+
+# Windows batch files
+*.bat
+
+cleanup.sh

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-70
+version: 2-1-6
 runtime: python27
 api_version: 1
 threadsafe: true

bs4/COPYING.txt

@@ -0,0 +1,26 @@
+Beautiful Soup is made available under the MIT license:
+
+ Copyright (c) 2004-2012 Leonard Richardson
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE, DAMMIT.
+
+Beautiful Soup incorporates code from the html5lib library, which is
+also made available under the MIT license.

bs4/__init__.py

@@ -0,0 +1,406 @@
+"""Beautiful Soup
+Elixir and Tonic
+"The Screen-Scraper's Friend"
+http://www.crummy.com/software/BeautifulSoup/
+
+Beautiful Soup uses a pluggable XML or HTML parser to parse a
+(possibly invalid) document into a tree representation. Beautiful Soup
+provides provides methods and Pythonic idioms that make it easy to
+navigate, search, and modify the parse tree.
+
+Beautiful Soup works with Python 2.6 and up. It works better if lxml
+and/or html5lib is installed.
+
+For more than you ever wanted to know about Beautiful Soup, see the
+documentation:
+http://www.crummy.com/software/BeautifulSoup/bs4/doc/
+"""
+
+__author__ = "Leonard Richardson (leonardr@segfault.org)"
+__version__ = "4.3.2"
+__copyright__ = "Copyright (c) 2004-2013 Leonard Richardson"
+__license__ = "MIT"
+
+__all__ = ['BeautifulSoup']
+
+import os
+import re
+import warnings
+
+from .builder import builder_registry, ParserRejectedMarkup
+from .dammit import UnicodeDammit
+from .element import (
+    CData,
+    Comment,
+    DEFAULT_OUTPUT_ENCODING,
+    Declaration,
+    Doctype,
+    NavigableString,
+    PageElement,
+    ProcessingInstruction,
+    ResultSet,
+    SoupStrainer,
+    Tag,
+    )
+
+# The very first thing we do is give a useful error if someone is
+# running this code under Python 3 without converting it.
+syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+
+class BeautifulSoup(Tag):
+    """
+    This class defines the basic interface called by the tree builders.
+
+    These methods will be called by the parser:
+      reset()
+      feed(markup)
+
+    The tree builder may call these methods from its feed() implementation:
+      handle_starttag(name, attrs) # See note about return value
+      handle_endtag(name)
+      handle_data(data) # Appends to the current data node
+      endData(containerClass=NavigableString) # Ends the current data node
+
+    No matter how complicated the underlying parser is, you should be
+    able to build a tree using 'start tag' events, 'end tag' events,
+    'data' events, and "done with data" events.
+
+    If you encounter an empty-element tag (aka a self-closing tag,
+    like HTML's <br> tag), call handle_starttag and then
+    handle_endtag.
+    """
+    ROOT_TAG_NAME = u'[document]'
+
+    # If the end-user gives no indication which tree builder they
+    # want, look for one with these features.
+    DEFAULT_BUILDER_FEATURES = ['html', 'fast']
+
+    ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
+
+    def __init__(self, markup="", features=None, builder=None,
+                 parse_only=None, from_encoding=None, **kwargs):
+        """The Soup object is initialized as the 'root tag', and the
+        provided markup (which can be a string or a file-like object)
+        is fed into the underlying parser."""
+
+        if 'convertEntities' in kwargs:
+            warnings.warn(
+                "BS4 does not respect the convertEntities argument to the "
+                "BeautifulSoup constructor. Entities are always converted "
+                "to Unicode characters.")
+
+        if 'markupMassage' in kwargs:
+            del kwargs['markupMassage']
+            warnings.warn(
+                "BS4 does not respect the markupMassage argument to the "
+                "BeautifulSoup constructor. The tree builder is responsible "
+                "for any necessary markup massage.")
+
+        if 'smartQuotesTo' in kwargs:
+            del kwargs['smartQuotesTo']
+            warnings.warn(
+                "BS4 does not respect the smartQuotesTo argument to the "
+                "BeautifulSoup constructor. Smart quotes are always converted "
+                "to Unicode characters.")
+
+        if 'selfClosingTags' in kwargs:
+            del kwargs['selfClosingTags']
+            warnings.warn(
+                "BS4 does not respect the selfClosingTags argument to the "
+                "BeautifulSoup constructor. The tree builder is responsible "
+                "for understanding self-closing tags.")
+
+        if 'isHTML' in kwargs:
+            del kwargs['isHTML']
+            warnings.warn(
+                "BS4 does not respect the isHTML argument to the "
+                "BeautifulSoup constructor. You can pass in features='html' "
+                "or features='xml' to get a builder capable of handling "
+                "one or the other.")
+
+        def deprecated_argument(old_name, new_name):
+            if old_name in kwargs:
+                warnings.warn(
+                    'The "%s" argument to the BeautifulSoup constructor '
+                    'has been renamed to "%s."' % (old_name, new_name))
+                value = kwargs[old_name]
+                del kwargs[old_name]
+                return value
+            return None
+
+        parse_only = parse_only or deprecated_argument(
+            "parseOnlyThese", "parse_only")
+
+        from_encoding = from_encoding or deprecated_argument(
+            "fromEncoding", "from_encoding")
+
+        if len(kwargs) > 0:
+            arg = kwargs.keys().pop()
+            raise TypeError(
+                "__init__() got an unexpected keyword argument '%s'" % arg)
+
+        if builder is None:
+            if isinstance(features, basestring):
+                features = [features]
+            if features is None or len(features) == 0:
+                features = self.DEFAULT_BUILDER_FEATURES
+            builder_class = builder_registry.lookup(*features)
+            if builder_class is None:
+                raise FeatureNotFound(
+                    "Couldn't find a tree builder with the features you "
+                    "requested: %s. Do you need to install a parser library?"
+                    % ",".join(features))
+            builder = builder_class()
+        self.builder = builder
+        self.is_xml = builder.is_xml
+        self.builder.soup = self
+
+        self.parse_only = parse_only
+
+        if hasattr(markup, 'read'):        # It's a file-type object.
+            markup = markup.read()
+        elif len(markup) <= 256:
+            # Print out warnings for a couple beginner problems
+            # involving passing non-markup to Beautiful Soup.
+            # Beautiful Soup will still parse the input as markup,
+            # just in case that's what the user really wants.
+            if (isinstance(markup, unicode)
+                and not os.path.supports_unicode_filenames):
+                possible_filename = markup.encode("utf8")
+            else:
+                possible_filename = markup
+            is_file = False
+            try:
+                is_file = os.path.exists(possible_filename)
+            except Exception, e:
+                # This is almost certainly a problem involving
+                # characters not valid in filenames on this
+                # system. Just let it go.
+                pass
+            if is_file:
+                warnings.warn(
+                    '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
+            if markup[:5] == "http:" or markup[:6] == "https:":
+                # TODO: This is ugly but I couldn't get it to work in
+                # Python 3 otherwise.
+                if ((isinstance(markup, bytes) and not b' ' in markup)
+                    or (isinstance(markup, unicode) and not u' ' in markup)):
+                    warnings.warn(
+                        '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
+
+        for (self.markup, self.original_encoding, self.declared_html_encoding,
+         self.contains_replacement_characters) in (
+            self.builder.prepare_markup(markup, from_encoding)):
+            self.reset()
+            try:
+                self._feed()
+                break
+            except ParserRejectedMarkup:
+                pass
+
+        # Clear out the markup and remove the builder's circular
+        # reference to this object.
+        self.markup = None
+        self.builder.soup = None
+
+    def _feed(self):
+        # Convert the document to Unicode.
+        self.builder.reset()
+
+        self.builder.feed(self.markup)
+        # Close out any unfinished strings and close all the open tags.
+        self.endData()
+        while self.currentTag.name != self.ROOT_TAG_NAME:
+            self.popTag()
+
+    def reset(self):
+        Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
+        self.hidden = 1
+        self.builder.reset()
+        self.current_data = []
+        self.currentTag = None
+        self.tagStack = []
+        self.preserve_whitespace_tag_stack = []
+        self.pushTag(self)
+
+    def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
+        """Create a new tag associated with this soup."""
+        return Tag(None, self.builder, name, namespace, nsprefix, attrs)
+
+    def new_string(self, s, subclass=NavigableString):
+        """Create a new NavigableString associated with this soup."""
+        navigable = subclass(s)
+        navigable.setup()
+        return navigable
+
+    def insert_before(self, successor):
+        raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
+
+    def insert_after(self, successor):
+        raise NotImplementedError("BeautifulSoup objects don't support insert_after().")
+
+    def popTag(self):
+        tag = self.tagStack.pop()
+        if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]:
+            self.preserve_whitespace_tag_stack.pop()
+        #print "Pop", tag.name
+        if self.tagStack:
+            self.currentTag = self.tagStack[-1]
+        return self.currentTag
+
+    def pushTag(self, tag):
+        #print "Push", tag.name
+        if self.currentTag:
+            self.currentTag.contents.append(tag)
+        self.tagStack.append(tag)
+        self.currentTag = self.tagStack[-1]
+        if tag.name in self.builder.preserve_whitespace_tags:
+            self.preserve_whitespace_tag_stack.append(tag)
+
+    def endData(self, containerClass=NavigableString):
+        if self.current_data:
+            current_data = u''.join(self.current_data)
+            # If whitespace is not preserved, and this string contains
+            # nothing but ASCII spaces, replace it with a single space
+            # or newline.
+            if not self.preserve_whitespace_tag_stack:
+                strippable = True
+                for i in current_data:
+                    if i not in self.ASCII_SPACES:
+                        strippable = False
+                        break
+                if strippable:
+                    if '\n' in current_data:
+                        current_data = '\n'
+                    else:
+                        current_data = ' '
+
+            # Reset the data collector.
+            self.current_data = []
+
+            # Should we add this string to the tree at all?
+            if self.parse_only and len(self.tagStack) <= 1 and \
+                   (not self.parse_only.text or \
+                    not self.parse_only.search(current_data)):
+                return
+
+            o = containerClass(current_data)
+            self.object_was_parsed(o)
+
+    def object_was_parsed(self, o, parent=None, most_recent_element=None):
+        """Add an object to the parse tree."""
+        parent = parent or self.currentTag
+        most_recent_element = most_recent_element or self._most_recent_element
+        o.setup(parent, most_recent_element)
+
+        if most_recent_element is not None:
+            most_recent_element.next_element = o
+        self._most_recent_element = o
+        parent.contents.append(o)
+
+    def _popToTag(self, name, nsprefix=None, inclusivePop=True):
+        """Pops the tag stack up to and including the most recent
+        instance of the given tag. If inclusivePop is false, pops the tag
+        stack up to but *not* including the most recent instqance of
+        the given tag."""
+        #print "Popping to %s" % name
+        if name == self.ROOT_TAG_NAME:
+            # The BeautifulSoup object itself can never be popped.
+            return
+
+        most_recently_popped = None
+
+        stack_size = len(self.tagStack)
+        for i in range(stack_size - 1, 0, -1):
+            t = self.tagStack[i]
+            if (name == t.name and nsprefix == t.prefix):
+                if inclusivePop:
+                    most_recently_popped = self.popTag()
+                break
+            most_recently_popped = self.popTag()
+
+        return most_recently_popped
+
+    def handle_starttag(self, name, namespace, nsprefix, attrs):
+        """Push a start tag on to the stack.
+
+        If this method returns None, the tag was rejected by the
+        SoupStrainer. You should proceed as if the tag had not occured
+        in the document. For instance, if this was a self-closing tag,
+        don't call handle_endtag.
+        """
+
+        # print "Start tag %s: %s" % (name, attrs)
+        self.endData()
+
+        if (self.parse_only and len(self.tagStack) <= 1
+            and (self.parse_only.text
+                 or not self.parse_only.search_tag(name, attrs))):
+            return None
+
+        tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
+                  self.currentTag, self._most_recent_element)
+        if tag is None:
+            return tag
+        if self._most_recent_element:
+            self._most_recent_element.next_element = tag
+        self._most_recent_element = tag
+        self.pushTag(tag)
+        return tag
+
+    def handle_endtag(self, name, nsprefix=None):
+        #print "End tag: " + name
+        self.endData()
+        self._popToTag(name, nsprefix)
+
+    def handle_data(self, data):
+        self.current_data.append(data)
+
+    def decode(self, pretty_print=False,
+               eventual_encoding=DEFAULT_OUTPUT_ENCODING,
+               formatter="minimal"):
+        """Returns a string or Unicode representation of this document.
+        To get Unicode, pass None for encoding."""
+
+        if self.is_xml:
+            # Print the XML declaration
+            encoding_part = ''
+            if eventual_encoding != None:
+                encoding_part = ' encoding="%s"' % eventual_encoding
+            prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
+        else:
+            prefix = u''
+        if not pretty_print:
+            indent_level = None
+        else:
+            indent_level = 0
+        return prefix + super(BeautifulSoup, self).decode(
+            indent_level, eventual_encoding, formatter)
+
+# Alias to make it easier to type import: 'from bs4 import _soup'
+_s = BeautifulSoup
+_soup = BeautifulSoup
+
+class BeautifulStoneSoup(BeautifulSoup):
+    """Deprecated interface to an XML parser."""
+
+    def __init__(self, *args, **kwargs):
+        kwargs['features'] = 'xml'
+        warnings.warn(
+            'The BeautifulStoneSoup class is deprecated. Instead of using '
+            'it, pass features="xml" into the BeautifulSoup constructor.')
+        super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
+
+
+class StopParsing(Exception):
+    pass
+
+class FeatureNotFound(ValueError):
+    pass
+
+
+#By default, act as an HTML pretty-printer.
+if __name__ == '__main__':
+    import sys
+    soup = BeautifulSoup(sys.stdin)
+    print soup.prettify()

bs4/builder/__init__.py

@@ -0,0 +1,321 @@
+from collections import defaultdict
+import itertools
+import sys
+from bs4.element import (
+    CharsetMetaAttributeValue,
+    ContentMetaAttributeValue,
+    whitespace_re
+    )
+
+__all__ = [
+    'HTMLTreeBuilder',
+    'SAXTreeBuilder',
+    'TreeBuilder',
+    'TreeBuilderRegistry',
+    ]
+
+# Some useful features for a TreeBuilder to have.
+FAST = 'fast'
+PERMISSIVE = 'permissive'
+STRICT = 'strict'
+XML = 'xml'
+HTML = 'html'
+HTML_5 = 'html5'
+
+
+class TreeBuilderRegistry(object):
+
+    def __init__(self):
+        self.builders_for_feature = defaultdict(list)
+        self.builders = []
+
+    def register(self, treebuilder_class):
+        """Register a treebuilder based on its advertised features."""
+        for feature in treebuilder_class.features:
+            self.builders_for_feature[feature].insert(0, treebuilder_class)
+        self.builders.insert(0, treebuilder_class)
+
+    def lookup(self, *features):
+        if len(self.builders) == 0:
+            # There are no builders at all.
+            return None
+
+        if len(features) == 0:
+            # They didn't ask for any features. Give them the most
+            # recently registered builder.
+            return self.builders[0]
+
+        # Go down the list of features in order, and eliminate any builders
+        # that don't match every feature.
+        features = list(features)
+        features.reverse()
+        candidates = None
+        candidate_set = None
+        while len(features) > 0:
+            feature = features.pop()
+            we_have_the_feature = self.builders_for_feature.get(feature, [])
+            if len(we_have_the_feature) > 0:
+                if candidates is None:
+                    candidates = we_have_the_feature
+                    candidate_set = set(candidates)
+                else:
+                    # Eliminate any candidates that don't have this feature.
+                    candidate_set = candidate_set.intersection(
+                        set(we_have_the_feature))
+
+        # The only valid candidates are the ones in candidate_set.
+        # Go through the original list of candidates and pick the first one
+        # that's in candidate_set.
+        if candidate_set is None:
+            return None
+        for candidate in candidates:
+            if candidate in candidate_set:
+                return candidate
+        return None
+
+# The BeautifulSoup class will take feature lists from developers and use them
+# to look up builders in this registry.
+builder_registry = TreeBuilderRegistry()
+
+class TreeBuilder(object):
+    """Turn a document into a Beautiful Soup object tree."""
+
+    features = []
+
+    is_xml = False
+    preserve_whitespace_tags = set()
+    empty_element_tags = None # A tag will be considered an empty-element
+                              # tag when and only when it has no contents.
+
+    # A value for these tag/attribute combinations is a space- or
+    # comma-separated list of CDATA, rather than a single CDATA.
+    cdata_list_attributes = {}
+
+
+    def __init__(self):
+        self.soup = None
+
+    def reset(self):
+        pass
+
+    def can_be_empty_element(self, tag_name):
+        """Might a tag with this name be an empty-element tag?
+
+        The final markup may or may not actually present this tag as
+        self-closing.
+
+        For instance: an HTMLBuilder does not consider a <p> tag to be
+        an empty-element tag (it's not in
+        HTMLBuilder.empty_element_tags). This means an empty <p> tag
+        will be presented as "<p></p>", not "<p />".
+
+        The default implementation has no opinion about which tags are
+        empty-element tags, so a tag will be presented as an
+        empty-element tag if and only if it has no contents.
+        "<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will
+        be left alone.
+        """
+        if self.empty_element_tags is None:
+            return True
+        return tag_name in self.empty_element_tags
+
+    def feed(self, markup):
+        raise NotImplementedError()
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        return markup, None, None, False
+
+    def test_fragment_to_document(self, fragment):
+        """Wrap an HTML fragment to make it look like a document.
+
+        Different parsers do this differently. For instance, lxml
+        introduces an empty <head> tag, and html5lib
+        doesn't. Abstracting this away lets us write simple tests
+        which run HTML fragments through the parser and compare the
+        results against other HTML fragments.
+
+        This method should not be used outside of tests.
+        """
+        return fragment
+
+    def set_up_substitutions(self, tag):
+        return False
+
+    def _replace_cdata_list_attribute_values(self, tag_name, attrs):
+        """Replaces class="foo bar" with class=["foo", "bar"]
+
+        Modifies its input in place.
+        """
+        if not attrs:
+            return attrs
+        if self.cdata_list_attributes:
+            universal = self.cdata_list_attributes.get('*', [])
+            tag_specific = self.cdata_list_attributes.get(
+                tag_name.lower(), None)
+            for attr in attrs.keys():
+                if attr in universal or (tag_specific and attr in tag_specific):
+                    # We have a "class"-type attribute whose string
+                    # value is a whitespace-separated list of
+                    # values. Split it into a list.
+                    value = attrs[attr]
+                    if isinstance(value, basestring):
+                        values = whitespace_re.split(value)
+                    else:
+                        # html5lib sometimes calls setAttributes twice
+                        # for the same tag when rearranging the parse
+                        # tree. On the second call the attribute value
+                        # here is already a list.  If this happens,
+                        # leave the value alone rather than trying to
+                        # split it again.
+                        values = value
+                    attrs[attr] = values
+        return attrs
+
+class SAXTreeBuilder(TreeBuilder):
+    """A Beautiful Soup treebuilder that listens for SAX events."""
+
+    def feed(self, markup):
+        raise NotImplementedError()
+
+    def close(self):
+        pass
+
+    def startElement(self, name, attrs):
+        attrs = dict((key[1], value) for key, value in list(attrs.items()))
+        #print "Start %s, %r" % (name, attrs)
+        self.soup.handle_starttag(name, attrs)
+
+    def endElement(self, name):
+        #print "End %s" % name
+        self.soup.handle_endtag(name)
+
+    def startElementNS(self, nsTuple, nodeName, attrs):
+        # Throw away (ns, nodeName) for now.
+        self.startElement(nodeName, attrs)
+
+    def endElementNS(self, nsTuple, nodeName):
+        # Throw away (ns, nodeName) for now.
+        self.endElement(nodeName)
+        #handler.endElementNS((ns, node.nodeName), node.nodeName)
+
+    def startPrefixMapping(self, prefix, nodeValue):
+        # Ignore the prefix for now.
+        pass
+
+    def endPrefixMapping(self, prefix):
+        # Ignore the prefix for now.
+        # handler.endPrefixMapping(prefix)
+        pass
+
+    def characters(self, content):
+        self.soup.handle_data(content)
+
+    def startDocument(self):
+        pass
+
+    def endDocument(self):
+        pass
+
+
+class HTMLTreeBuilder(TreeBuilder):
+    """This TreeBuilder knows facts about HTML.
+
+    Such as which tags are empty-element tags.
+    """
+
+    preserve_whitespace_tags = set(['pre', 'textarea'])
+    empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
+                              'spacer', 'link', 'frame', 'base'])
+
+    # The HTML standard defines these attributes as containing a
+    # space-separated list of values, not a single value. That is,
+    # class="foo bar" means that the 'class' attribute has two values,
+    # 'foo' and 'bar', not the single value 'foo bar'.  When we
+    # encounter one of these attributes, we will parse its value into
+    # a list of values if possible. Upon output, the list will be
+    # converted back into a string.
+    cdata_list_attributes = {
+        "*" : ['class', 'accesskey', 'dropzone'],
+        "a" : ['rel', 'rev'],
+        "link" :  ['rel', 'rev'],
+        "td" : ["headers"],
+        "th" : ["headers"],
+        "td" : ["headers"],
+        "form" : ["accept-charset"],
+        "object" : ["archive"],
+
+        # These are HTML5 specific, as are *.accesskey and *.dropzone above.
+        "area" : ["rel"],
+        "icon" : ["sizes"],
+        "iframe" : ["sandbox"],
+        "output" : ["for"],
+        }
+
+    def set_up_substitutions(self, tag):
+        # We are only interested in <meta> tags
+        if tag.name != 'meta':
+            return False
+
+        http_equiv = tag.get('http-equiv')
+        content = tag.get('content')
+        charset = tag.get('charset')
+
+        # We are interested in <meta> tags that say what encoding the
+        # document was originally in. This means HTML 5-style <meta>
+        # tags that provide the "charset" attribute. It also means
+        # HTML 4-style <meta> tags that provide the "content"
+        # attribute and have "http-equiv" set to "content-type".
+        #
+        # In both cases we will replace the value of the appropriate
+        # attribute with a standin object that can take on any
+        # encoding.
+        meta_encoding = None
+        if charset is not None:
+            # HTML 5 style:
+            # <meta charset="utf8">
+            meta_encoding = charset
+            tag['charset'] = CharsetMetaAttributeValue(charset)
+
+        elif (content is not None and http_equiv is not None
+              and http_equiv.lower() == 'content-type'):
+            # HTML 4 style:
+            # <meta http-equiv="content-type" content="text/html; charset=utf8">
+            tag['content'] = ContentMetaAttributeValue(content)
+
+        return (meta_encoding is not None)
+
+def register_treebuilders_from(module):
+    """Copy TreeBuilders from the given module into this module."""
+    # I'm fairly sure this is not the best way to do this.
+    this_module = sys.modules['bs4.builder']
+    for name in module.__all__:
+        obj = getattr(module, name)
+
+        if issubclass(obj, TreeBuilder):
+            setattr(this_module, name, obj)
+            this_module.__all__.append(name)
+            # Register the builder while we're at it.
+            this_module.builder_registry.register(obj)
+
+class ParserRejectedMarkup(Exception):
+    pass
+
+# Builders are registered in reverse order of priority, so that custom
+# builder registrations will take precedence. In general, we want lxml
+# to take precedence over html5lib, because it's faster. And we only
+# want to use HTMLParser as a last result.
+from . import _htmlparser
+register_treebuilders_from(_htmlparser)
+try:
+    from . import _html5lib
+    register_treebuilders_from(_html5lib)
+except ImportError:
+    # They don't have html5lib installed.
+    pass
+try:
+    from . import _lxml
+    register_treebuilders_from(_lxml)
+except ImportError:
+    # They don't have lxml installed.
+    pass

bs4/builder/_html5lib.py

@@ -0,0 +1,285 @@
+__all__ = [
+    'HTML5TreeBuilder',
+    ]
+
+import warnings
+from bs4.builder import (
+    PERMISSIVE,
+    HTML,
+    HTML_5,
+    HTMLTreeBuilder,
+    )
+from bs4.element import NamespacedAttribute
+import html5lib
+from html5lib.constants import namespaces
+from bs4.element import (
+    Comment,
+    Doctype,
+    NavigableString,
+    Tag,
+    )
+
+class HTML5TreeBuilder(HTMLTreeBuilder):
+    """Use html5lib to build a tree."""
+
+    features = ['html5lib', PERMISSIVE, HTML_5, HTML]
+
+    def prepare_markup(self, markup, user_specified_encoding):
+        # Store the user-specified encoding for use later on.
+        self.user_specified_encoding = user_specified_encoding
+        yield (markup, None, None, False)
+
+    # These methods are defined by Beautiful Soup.
+    def feed(self, markup):
+        if self.soup.parse_only is not None:
+            warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
+        parser = html5lib.HTMLParser(tree=self.create_treebuilder)
+        doc = parser.parse(markup, encoding=self.user_specified_encoding)
+
+        # Set the character encoding detected by the tokenizer.
+        if isinstance(markup, unicode):
+            # We need to special-case this because html5lib sets
+            # charEncoding to UTF-8 if it gets Unicode input.
+            doc.original_encoding = None
+        else:
+            doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
+
+    def create_treebuilder(self, namespaceHTMLElements):
+        self.underlying_builder = TreeBuilderForHtml5lib(
+            self.soup, namespaceHTMLElements)
+        return self.underlying_builder
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return u'<html><head></head><body>%s</body></html>' % fragment
+
+
+class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
+
+    def __init__(self, soup, namespaceHTMLElements):
+        self.soup = soup
+        super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
+
+    def documentClass(self):
+        self.soup.reset()
+        return Element(self.soup, self.soup, None)
+
+    def insertDoctype(self, token):
+        name = token["name"]
+        publicId = token["publicId"]
+        systemId = token["systemId"]
+
+        doctype = Doctype.for_name_and_ids(name, publicId, systemId)
+        self.soup.object_was_parsed(doctype)
+
+    def elementClass(self, name, namespace):
+        tag = self.soup.new_tag(name, namespace)
+        return Element(tag, self.soup, namespace)
+
+    def commentClass(self, data):
+        return TextNode(Comment(data), self.soup)
+
+    def fragmentClass(self):
+        self.soup = BeautifulSoup("")
+        self.soup.name = "[document_fragment]"
+        return Element(self.soup, self.soup, None)
+
+    def appendChild(self, node):
+        # XXX This code is not covered by the BS4 tests.
+        self.soup.append(node.element)
+
+    def getDocument(self):
+        return self.soup
+
+    def getFragment(self):
+        return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
+
+class AttrList(object):
+    def __init__(self, element):
+        self.element = element
+        self.attrs = dict(self.element.attrs)
+    def __iter__(self):
+        return list(self.attrs.items()).__iter__()
+    def __setitem__(self, name, value):
+        "set attr", name, value
+        self.element[name] = value
+    def items(self):
+        return list(self.attrs.items())
+    def keys(self):
+        return list(self.attrs.keys())
+    def __len__(self):
+        return len(self.attrs)
+    def __getitem__(self, name):
+        return self.attrs[name]
+    def __contains__(self, name):
+        return name in list(self.attrs.keys())
+
+
+class Element(html5lib.treebuilders._base.Node):
+    def __init__(self, element, soup, namespace):
+        html5lib.treebuilders._base.Node.__init__(self, element.name)
+        self.element = element
+        self.soup = soup
+        self.namespace = namespace
+
+    def appendChild(self, node):
+        string_child = child = None
+        if isinstance(node, basestring):
+            # Some other piece of code decided to pass in a string
+            # instead of creating a TextElement object to contain the
+            # string.
+            string_child = child = node
+        elif isinstance(node, Tag):
+            # Some other piece of code decided to pass in a Tag
+            # instead of creating an Element object to contain the
+            # Tag.
+            child = node
+        elif node.element.__class__ == NavigableString:
+            string_child = child = node.element
+        else:
+            child = node.element
+
+        if not isinstance(child, basestring) and child.parent is not None:
+            node.element.extract()
+
+        if (string_child and self.element.contents
+            and self.element.contents[-1].__class__ == NavigableString):
+            # We are appending a string onto another string.
+            # TODO This has O(n^2) performance, for input like
+            # "a</a>a</a>a</a>..."
+            old_element = self.element.contents[-1]
+            new_element = self.soup.new_string(old_element + string_child)
+            old_element.replace_with(new_element)
+            self.soup._most_recent_element = new_element
+        else:
+            if isinstance(node, basestring):
+                # Create a brand new NavigableString from this string.
+                child = self.soup.new_string(node)
+
+            # Tell Beautiful Soup to act as if it parsed this element
+            # immediately after the parent's last descendant. (Or
+            # immediately after the parent, if it has no children.)
+            if self.element.contents:
+                most_recent_element = self.element._last_descendant(False)
+            else:
+                most_recent_element = self.element
+
+            self.soup.object_was_parsed(
+                child, parent=self.element,
+                most_recent_element=most_recent_element)
+
+    def getAttributes(self):
+        return AttrList(self.element)
+
+    def setAttributes(self, attributes):
+        if attributes is not None and len(attributes) > 0:
+
+            converted_attributes = []
+            for name, value in list(attributes.items()):
+                if isinstance(name, tuple):
+                    new_name = NamespacedAttribute(*name)
+                    del attributes[name]
+                    attributes[new_name] = value
+
+            self.soup.builder._replace_cdata_list_attribute_values(
+                self.name, attributes)
+            for name, value in attributes.items():
+                self.element[name] = value
+
+            # The attributes may contain variables that need substitution.
+            # Call set_up_substitutions manually.
+            #
+            # The Tag constructor called this method when the Tag was created,
+            # but we just set/changed the attributes, so call it again.
+            self.soup.builder.set_up_substitutions(self.element)
+    attributes = property(getAttributes, setAttributes)
+
+    def insertText(self, data, insertBefore=None):
+        if insertBefore:
+            text = TextNode(self.soup.new_string(data), self.soup)
+            self.insertBefore(data, insertBefore)
+        else:
+            self.appendChild(data)
+
+    def insertBefore(self, node, refNode):
+        index = self.element.index(refNode.element)
+        if (node.element.__class__ == NavigableString and self.element.contents
+            and self.element.contents[index-1].__class__ == NavigableString):
+            # (See comments in appendChild)
+            old_node = self.element.contents[index-1]
+            new_str = self.soup.new_string(old_node + node.element)
+            old_node.replace_with(new_str)
+        else:
+            self.element.insert(index, node.element)
+            node.parent = self
+
+    def removeChild(self, node):
+        node.element.extract()
+
+    def reparentChildren(self, new_parent):
+        """Move all of this tag's children into another tag."""
+        element = self.element
+        new_parent_element = new_parent.element
+        # Determine what this tag's next_element will be once all the children
+        # are removed.
+        final_next_element = element.next_sibling
+
+        new_parents_last_descendant = new_parent_element._last_descendant(False, False)
+        if len(new_parent_element.contents) > 0:
+            # The new parent already contains children. We will be
+            # appending this tag's children to the end.
+            new_parents_last_child = new_parent_element.contents[-1]
+            new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
+        else:
+            # The new parent contains no children.
+            new_parents_last_child = None
+            new_parents_last_descendant_next_element = new_parent_element.next_element
+
+        to_append = element.contents
+        append_after = new_parent.element.contents
+        if len(to_append) > 0:
+            # Set the first child's previous_element and previous_sibling
+            # to elements within the new parent
+            first_child = to_append[0]
+            first_child.previous_element = new_parents_last_descendant
+            first_child.previous_sibling = new_parents_last_child
+
+            # Fix the last child's next_element and next_sibling
+            last_child = to_append[-1]
+            last_child.next_element = new_parents_last_descendant_next_element
+            last_child.next_sibling = None
+
+        for child in to_append:
+            child.parent = new_parent_element
+            new_parent_element.contents.append(child)
+
+        # Now that this element has no children, change its .next_element.
+        element.contents = []
+        element.next_element = final_next_element
+
+    def cloneNode(self):
+        tag = self.soup.new_tag(self.element.name, self.namespace)
+        node = Element(tag, self.soup, self.namespace)
+        for key,value in self.attributes:
+            node.attributes[key] = value
+        return node
+
+    def hasContent(self):
+        return self.element.contents
+
+    def getNameTuple(self):
+        if self.namespace == None:
+            return namespaces["html"], self.name
+        else:
+            return self.namespace, self.name
+
+    nameTuple = property(getNameTuple)
+
+class TextNode(Element):
+    def __init__(self, element, soup):
+        html5lib.treebuilders._base.Node.__init__(self, None)
+        self.element = element
+        self.soup = soup
+
+    def cloneNode(self):
+        raise NotImplementedError

bs4/builder/_htmlparser.py

@@ -0,0 +1,258 @@
+"""Use the HTMLParser library to parse HTML files that aren't too bad."""
+
+__all__ = [
+    'HTMLParserTreeBuilder',
+    ]
+
+from HTMLParser import (
+    HTMLParser,
+    HTMLParseError,
+    )
+import sys
+import warnings
+
+# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
+# argument, which we'd like to set to False. Unfortunately,
+# http://bugs.python.org/issue13273 makes strict=True a better bet
+# before Python 3.2.3.
+#
+# At the end of this file, we monkeypatch HTMLParser so that
+# strict=True works well on Python 3.2.2.
+major, minor, release = sys.version_info[:3]
+CONSTRUCTOR_TAKES_STRICT = (
+    major > 3
+    or (major == 3 and minor > 2)
+    or (major == 3 and minor == 2 and release >= 3))
+
+from bs4.element import (
+    CData,
+    Comment,
+    Declaration,
+    Doctype,
+    ProcessingInstruction,
+    )
+from bs4.dammit import EntitySubstitution, UnicodeDammit
+
+from bs4.builder import (
+    HTML,
+    HTMLTreeBuilder,
+    STRICT,
+    )
+
+
+HTMLPARSER = 'html.parser'
+
+class BeautifulSoupHTMLParser(HTMLParser):
+    def handle_starttag(self, name, attrs):
+        # XXX namespace
+        attr_dict = {}
+        for key, value in attrs:
+            # Change None attribute values to the empty string
+            # for consistency with the other tree builders.
+            if value is None:
+                value = ''
+            attr_dict[key] = value
+            attrvalue = '""'
+        self.soup.handle_starttag(name, None, None, attr_dict)
+
+    def handle_endtag(self, name):
+        self.soup.handle_endtag(name)
+
+    def handle_data(self, data):
+        self.soup.handle_data(data)
+
+    def handle_charref(self, name):
+        # XXX workaround for a bug in HTMLParser. Remove this once
+        # it's fixed.
+        if name.startswith('x'):
+            real_name = int(name.lstrip('x'), 16)
+        elif name.startswith('X'):
+            real_name = int(name.lstrip('X'), 16)
+        else:
+            real_name = int(name)
+
+        try:
+            data = unichr(real_name)
+        except (ValueError, OverflowError), e:
+            data = u"\N{REPLACEMENT CHARACTER}"
+
+        self.handle_data(data)
+
+    def handle_entityref(self, name):
+        character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
+        if character is not None:
+            data = character
+        else:
+            data = "&%s;" % name
+        self.handle_data(data)
+
+    def handle_comment(self, data):
+        self.soup.endData()
+        self.soup.handle_data(data)
+        self.soup.endData(Comment)
+
+    def handle_decl(self, data):
+        self.soup.endData()
+        if data.startswith("DOCTYPE "):
+            data = data[len("DOCTYPE "):]
+        elif data == 'DOCTYPE':
+            # i.e. "<!DOCTYPE>"
+            data = ''
+        self.soup.handle_data(data)
+        self.soup.endData(Doctype)
+
+    def unknown_decl(self, data):
+        if data.upper().startswith('CDATA['):
+            cls = CData
+            data = data[len('CDATA['):]
+        else:
+            cls = Declaration
+        self.soup.endData()
+        self.soup.handle_data(data)
+        self.soup.endData(cls)
+
+    def handle_pi(self, data):
+        self.soup.endData()
+        if data.endswith("?") and data.lower().startswith("xml"):
+            # "An XHTML processing instruction using the trailing '?'
+            # will cause the '?' to be included in data." - HTMLParser
+            # docs.
+            #
+            # Strip the question mark so we don't end up with two
+            # question marks.
+            data = data[:-1]
+        self.soup.handle_data(data)
+        self.soup.endData(ProcessingInstruction)
+
+
+class HTMLParserTreeBuilder(HTMLTreeBuilder):
+
+    is_xml = False
+    features = [HTML, STRICT, HTMLPARSER]
+
+    def __init__(self, *args, **kwargs):
+        if CONSTRUCTOR_TAKES_STRICT:
+            kwargs['strict'] = False
+        self.parser_args = (args, kwargs)
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        """
+        :return: A 4-tuple (markup, original encoding, encoding
+        declared within markup, whether any characters had to be
+        replaced with REPLACEMENT CHARACTER).
+        """
+        if isinstance(markup, unicode):
+            yield (markup, None, None, False)
+            return
+
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
+        yield (dammit.markup, dammit.original_encoding,
+               dammit.declared_html_encoding,
+               dammit.contains_replacement_characters)
+
+    def feed(self, markup):
+        args, kwargs = self.parser_args
+        parser = BeautifulSoupHTMLParser(*args, **kwargs)
+        parser.soup = self.soup
+        try:
+            parser.feed(markup)
+        except HTMLParseError, e:
+            warnings.warn(RuntimeWarning(
+                "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
+            raise e
+
+# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
+# 3.2.3 code. This ensures they don't treat markup like <p></p> as a
+# string.
+#
+# XXX This code can be removed once most Python 3 users are on 3.2.3.
+if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
+    import re
+    attrfind_tolerant = re.compile(
+        r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
+        r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
+    HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
+
+    locatestarttagend = re.compile(r"""
+  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
+  (?:\s+                             # whitespace before attribute name
+    (?:[a-zA-Z_][-.:a-zA-Z0-9_]*     # attribute name
+      (?:\s*=\s*                     # value indicator
+        (?:'[^']*'                   # LITA-enclosed value
+          |\"[^\"]*\"                # LIT-enclosed value
+          |[^'\">\s]+                # bare value
+         )
+       )?
+     )
+   )*
+  \s*                                # trailing whitespace
+""", re.VERBOSE)
+    BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
+
+    from html.parser import tagfind, attrfind
+
+    def parse_starttag(self, i):
+        self.__starttag_text = None
+        endpos = self.check_for_whole_start_tag(i)
+        if endpos < 0:
+            return endpos
+        rawdata = self.rawdata
+        self.__starttag_text = rawdata[i:endpos]
+
+        # Now parse the data between i+1 and j into a tag and attrs
+        attrs = []
+        match = tagfind.match(rawdata, i+1)
+        assert match, 'unexpected call to parse_starttag()'
+        k = match.end()
+        self.lasttag = tag = rawdata[i+1:k].lower()
+        while k < endpos:
+            if self.strict:
+                m = attrfind.match(rawdata, k)
+            else:
+                m = attrfind_tolerant.match(rawdata, k)
+            if not m:
+                break
+            attrname, rest, attrvalue = m.group(1, 2, 3)
+            if not rest:
+                attrvalue = None
+            elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
+                 attrvalue[:1] == '"' == attrvalue[-1:]:
+                attrvalue = attrvalue[1:-1]
+            if attrvalue:
+                attrvalue = self.unescape(attrvalue)
+            attrs.append((attrname.lower(), attrvalue))
+            k = m.end()
+
+        end = rawdata[k:endpos].strip()
+        if end not in (">", "/>"):
+            lineno, offset = self.getpos()
+            if "\n" in self.__starttag_text:
+                lineno = lineno + self.__starttag_text.count("\n")
+                offset = len(self.__starttag_text) \
+                         - self.__starttag_text.rfind("\n")
+            else:
+                offset = offset + len(self.__starttag_text)
+            if self.strict:
+                self.error("junk characters in start tag: %r"
+                           % (rawdata[k:endpos][:20],))
+            self.handle_data(rawdata[i:endpos])
+            return endpos
+        if end.endswith('/>'):
+            # XHTML-style empty tag: <span attr="value" />
+            self.handle_startendtag(tag, attrs)
+        else:
+            self.handle_starttag(tag, attrs)
+            if tag in self.CDATA_CONTENT_ELEMENTS:
+                self.set_cdata_mode(tag)
+        return endpos
+
+    def set_cdata_mode(self, elem):
+        self.cdata_elem = elem.lower()
+        self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
+
+    BeautifulSoupHTMLParser.parse_starttag = parse_starttag
+    BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
+
+    CONSTRUCTOR_TAKES_STRICT = True

bs4/builder/_lxml.py

@@ -0,0 +1,233 @@
+__all__ = [
+    'LXMLTreeBuilderForXML',
+    'LXMLTreeBuilder',
+    ]
+
+from io import BytesIO
+from StringIO import StringIO
+import collections
+from lxml import etree
+from bs4.element import Comment, Doctype, NamespacedAttribute
+from bs4.builder import (
+    FAST,
+    HTML,
+    HTMLTreeBuilder,
+    PERMISSIVE,
+    ParserRejectedMarkup,
+    TreeBuilder,
+    XML)
+from bs4.dammit import EncodingDetector
+
+LXML = 'lxml'
+
+class LXMLTreeBuilderForXML(TreeBuilder):
+    DEFAULT_PARSER_CLASS = etree.XMLParser
+
+    is_xml = True
+
+    # Well, it's permissive by XML parser standards.
+    features = [LXML, XML, FAST, PERMISSIVE]
+
+    CHUNK_SIZE = 512
+
+    # This namespace mapping is specified in the XML Namespace
+    # standard.
+    DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"}
+
+    def default_parser(self, encoding):
+        # This can either return a parser object or a class, which
+        # will be instantiated with default arguments.
+        if self._default_parser is not None:
+            return self._default_parser
+        return etree.XMLParser(
+            target=self, strip_cdata=False, recover=True, encoding=encoding)
+
+    def parser_for(self, encoding):
+        # Use the default parser.
+        parser = self.default_parser(encoding)
+
+        if isinstance(parser, collections.Callable):
+            # Instantiate the parser with default arguments
+            parser = parser(target=self, strip_cdata=False, encoding=encoding)
+        return parser
+
+    def __init__(self, parser=None, empty_element_tags=None):
+        # TODO: Issue a warning if parser is present but not a
+        # callable, since that means there's no way to create new
+        # parsers for different encodings.
+        self._default_parser = parser
+        if empty_element_tags is not None:
+            self.empty_element_tags = set(empty_element_tags)
+        self.soup = None
+        self.nsmaps = [self.DEFAULT_NSMAPS]
+
+    def _getNsTag(self, tag):
+        # Split the namespace URL out of a fully-qualified lxml tag
+        # name. Copied from lxml's src/lxml/sax.py.
+        if tag[0] == '{':
+            return tuple(tag[1:].split('}', 1))
+        else:
+            return (None, tag)
+
+    def prepare_markup(self, markup, user_specified_encoding=None,
+                       document_declared_encoding=None):
+        """
+        :yield: A series of 4-tuples.
+         (markup, encoding, declared encoding,
+          has undergone character replacement)
+
+        Each 4-tuple represents a strategy for parsing the document.
+        """
+        if isinstance(markup, unicode):
+            # We were given Unicode. Maybe lxml can parse Unicode on
+            # this system?
+            yield markup, None, document_declared_encoding, False
+
+        if isinstance(markup, unicode):
+            # No, apparently not. Convert the Unicode to UTF-8 and
+            # tell lxml to parse it as UTF-8.
+            yield (markup.encode("utf8"), "utf8",
+                   document_declared_encoding, False)
+
+        # Instead of using UnicodeDammit to convert the bytestring to
+        # Unicode using different encodings, use EncodingDetector to
+        # iterate over the encodings, and tell lxml to try to parse
+        # the document as each one in turn.
+        is_html = not self.is_xml
+        try_encodings = [user_specified_encoding, document_declared_encoding]
+        detector = EncodingDetector(markup, try_encodings, is_html)
+        for encoding in detector.encodings:
+            yield (detector.markup, encoding, document_declared_encoding, False)
+
+    def feed(self, markup):
+        if isinstance(markup, bytes):
+            markup = BytesIO(markup)
+        elif isinstance(markup, unicode):
+            markup = StringIO(markup)
+
+        # Call feed() at least once, even if the markup is empty,
+        # or the parser won't be initialized.
+        data = markup.read(self.CHUNK_SIZE)
+        try:
+            self.parser = self.parser_for(self.soup.original_encoding)
+            self.parser.feed(data)
+            while len(data) != 0:
+                # Now call feed() on the rest of the data, chunk by chunk.
+                data = markup.read(self.CHUNK_SIZE)
+                if len(data) != 0:
+                    self.parser.feed(data)
+            self.parser.close()
+        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
+            raise ParserRejectedMarkup(str(e))
+
+    def close(self):
+        self.nsmaps = [self.DEFAULT_NSMAPS]
+
+    def start(self, name, attrs, nsmap={}):
+        # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
+        attrs = dict(attrs)
+        nsprefix = None
+        # Invert each namespace map as it comes in.
+        if len(self.nsmaps) > 1:
+            # There are no new namespaces for this tag, but
+            # non-default namespaces are in play, so we need a
+            # separate tag stack to know when they end.
+            self.nsmaps.append(None)
+        elif len(nsmap) > 0:
+            # A new namespace mapping has come into play.
+            inverted_nsmap = dict((value, key) for key, value in nsmap.items())
+            self.nsmaps.append(inverted_nsmap)
+            # Also treat the namespace mapping as a set of attributes on the
+            # tag, so we can recreate it later.
+            attrs = attrs.copy()
+            for prefix, namespace in nsmap.items():
+                attribute = NamespacedAttribute(
+                    "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
+                attrs[attribute] = namespace
+
+        # Namespaces are in play. Find any attributes that came in
+        # from lxml with namespaces attached to their names, and
+        # turn then into NamespacedAttribute objects.
+        new_attrs = {}
+        for attr, value in attrs.items():
+            namespace, attr = self._getNsTag(attr)
+            if namespace is None:
+                new_attrs[attr] = value
+            else:
+                nsprefix = self._prefix_for_namespace(namespace)
+                attr = NamespacedAttribute(nsprefix, attr, namespace)
+                new_attrs[attr] = value
+        attrs = new_attrs
+
+        namespace, name = self._getNsTag(name)
+        nsprefix = self._prefix_for_namespace(namespace)
+        self.soup.handle_starttag(name, namespace, nsprefix, attrs)
+
+    def _prefix_for_namespace(self, namespace):
+        """Find the currently active prefix for the given namespace."""
+        if namespace is None:
+            return None
+        for inverted_nsmap in reversed(self.nsmaps):
+            if inverted_nsmap is not None and namespace in inverted_nsmap:
+                return inverted_nsmap[namespace]
+        return None
+
+    def end(self, name):
+        self.soup.endData()
+        completed_tag = self.soup.tagStack[-1]
+        namespace, name = self._getNsTag(name)
+        nsprefix = None
+        if namespace is not None:
+            for inverted_nsmap in reversed(self.nsmaps):
+                if inverted_nsmap is not None and namespace in inverted_nsmap:
+                    nsprefix = inverted_nsmap[namespace]
+                    break
+        self.soup.handle_endtag(name, nsprefix)
+        if len(self.nsmaps) > 1:
+            # This tag, or one of its parents, introduced a namespace
+            # mapping, so pop it off the stack.
+            self.nsmaps.pop()
+
+    def pi(self, target, data):
+        pass
+
+    def data(self, content):
+        self.soup.handle_data(content)
+
+    def doctype(self, name, pubid, system):
+        self.soup.endData()
+        doctype = Doctype.for_name_and_ids(name, pubid, system)
+        self.soup.object_was_parsed(doctype)
+
+    def comment(self, content):
+        "Handle comments as Comment objects."
+        self.soup.endData()
+        self.soup.handle_data(content)
+        self.soup.endData(Comment)
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
+
+
+class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
+
+    features = [LXML, HTML, FAST, PERMISSIVE]
+    is_xml = False
+
+    def default_parser(self, encoding):
+        return etree.HTMLParser
+
+    def feed(self, markup):
+        encoding = self.soup.original_encoding
+        try:
+            self.parser = self.parser_for(encoding)
+            self.parser.feed(markup)
+            self.parser.close()
+        except (UnicodeDecodeError, LookupError, etree.ParserError), e:
+            raise ParserRejectedMarkup(str(e))
+
+
+    def test_fragment_to_document(self, fragment):
+        """See `TreeBuilder`."""
+        return u'<html><body>%s</body></html>' % fragment

bs4/dammit.py

@@ -0,0 +1,829 @@
+# -*- coding: utf-8 -*-
+"""Beautiful Soup bonus library: Unicode, Dammit
+
+This library converts a bytestream to Unicode through any means
+necessary. It is heavily based on code from Mark Pilgrim's Universal
+Feed Parser. It works best on XML and XML, but it does not rewrite the
+XML or HTML to reflect a new encoding; that's the tree builder's job.
+"""
+
+import codecs
+from htmlentitydefs import codepoint2name
+import re
+import logging
+import string
+
+# Import a library to autodetect character encodings.
+chardet_type = None
+try:
+    # First try the fast C implementation.
+    #  PyPI package: cchardet
+    import cchardet
+    def chardet_dammit(s):
+        return cchardet.detect(s)['encoding']
+except ImportError:
+    try:
+        # Fall back to the pure Python implementation
+        #  Debian package: python-chardet
+        #  PyPI package: chardet
+        import chardet
+        def chardet_dammit(s):
+            return chardet.detect(s)['encoding']
+        #import chardet.constants
+        #chardet.constants._debug = 1
+    except ImportError:
+        # No chardet available.
+        def chardet_dammit(s):
+            return None
+
+# Available from http://cjkpython.i18n.org/.
+try:
+    import iconv_codec
+except ImportError:
+    pass
+
+xml_encoding_re = re.compile(
+    '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
+html_meta_re = re.compile(
+    '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
+
+class EntitySubstitution(object):
+
+    """Substitute XML or HTML entities for the corresponding characters."""
+
+    def _populate_class_variables():
+        lookup = {}
+        reverse_lookup = {}
+        characters_for_re = []
+        for codepoint, name in list(codepoint2name.items()):
+            character = unichr(codepoint)
+            if codepoint != 34:
+                # There's no point in turning the quotation mark into
+                # &quot;, unless it happens within an attribute value, which
+                # is handled elsewhere.
+                characters_for_re.append(character)
+                lookup[character] = name
+            # But we do want to turn &quot; into the quotation mark.
+            reverse_lookup[name] = character
+        re_definition = "[%s]" % "".join(characters_for_re)
+        return lookup, reverse_lookup, re.compile(re_definition)
+    (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
+     CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
+
+    CHARACTER_TO_XML_ENTITY = {
+        "'": "apos",
+        '"': "quot",
+        "&": "amp",
+        "<": "lt",
+        ">": "gt",
+        }
+
+    BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
+                                           "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
+                                           ")")
+
+    AMPERSAND_OR_BRACKET = re.compile("([<>&])")
+
+    @classmethod
+    def _substitute_html_entity(cls, matchobj):
+        entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
+        return "&%s;" % entity
+
+    @classmethod
+    def _substitute_xml_entity(cls, matchobj):
+        """Used with a regular expression to substitute the
+        appropriate XML entity for an XML special character."""
+        entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
+        return "&%s;" % entity
+
+    @classmethod
+    def quoted_attribute_value(self, value):
+        """Make a value into a quoted XML attribute, possibly escaping it.
+
+         Most strings will be quoted using double quotes.
+
+          Bob's Bar -> "Bob's Bar"
+
+         If a string contains double quotes, it will be quoted using
+         single quotes.
+
+          Welcome to "my bar" -> 'Welcome to "my bar"'
+
+         If a string contains both single and double quotes, the
+         double quotes will be escaped, and the string will be quoted
+         using double quotes.
+
+          Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
+        """
+        quote_with = '"'
+        if '"' in value:
+            if "'" in value:
+                # The string contains both single and double
+                # quotes.  Turn the double quotes into
+                # entities. We quote the double quotes rather than
+                # the single quotes because the entity name is
+                # "&quot;" whether this is HTML or XML.  If we
+                # quoted the single quotes, we'd have to decide
+                # between &apos; and &squot;.
+                replace_with = "&quot;"
+                value = value.replace('"', replace_with)
+            else:
+                # There are double quotes but no single quotes.
+                # We can use single quotes to quote the attribute.
+                quote_with = "'"
+        return quote_with + value + quote_with
+
+    @classmethod
+    def substitute_xml(cls, value, make_quoted_attribute=False):
+        """Substitute XML entities for special XML characters.
+
+        :param value: A string to be substituted. The less-than sign
+          will become &lt;, the greater-than sign will become &gt;,
+          and any ampersands will become &amp;. If you want ampersands
+          that appear to be part of an entity definition to be left
+          alone, use substitute_xml_containing_entities() instead.
+
+        :param make_quoted_attribute: If True, then the string will be
+         quoted, as befits an attribute value.
+        """
+        # Escape angle brackets and ampersands.
+        value = cls.AMPERSAND_OR_BRACKET.sub(
+            cls._substitute_xml_entity, value)
+
+        if make_quoted_attribute:
+            value = cls.quoted_attribute_value(value)
+        return value
+
+    @classmethod
+    def substitute_xml_containing_entities(
+        cls, value, make_quoted_attribute=False):
+        """Substitute XML entities for special XML characters.
+
+        :param value: A string to be substituted. The less-than sign will
+          become &lt;, the greater-than sign will become &gt;, and any
+          ampersands that are not part of an entity defition will
+          become &amp;.
+
+        :param make_quoted_attribute: If True, then the string will be
+         quoted, as befits an attribute value.
+        """
+        # Escape angle brackets, and ampersands that aren't part of
+        # entities.
+        value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
+            cls._substitute_xml_entity, value)
+
+        if make_quoted_attribute:
+            value = cls.quoted_attribute_value(value)
+        return value
+
+    @classmethod
+    def substitute_html(cls, s):
+        """Replace certain Unicode characters with named HTML entities.
+
+        This differs from data.encode(encoding, 'xmlcharrefreplace')
+        in that the goal is to make the result more readable (to those
+        with ASCII displays) rather than to recover from
+        errors. There's absolutely nothing wrong with a UTF-8 string
+        containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
+        character with "&eacute;" will make it more readable to some
+        people.
+        """
+        return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
+            cls._substitute_html_entity, s)
+
+
+class EncodingDetector:
+    """Suggests a number of possible encodings for a bytestring.
+
+    Order of precedence:
+
+    1. Encodings you specifically tell EncodingDetector to try first
+    (the override_encodings argument to the constructor).
+
+    2. An encoding declared within the bytestring itself, either in an
+    XML declaration (if the bytestring is to be interpreted as an XML
+    document), or in a <meta> tag (if the bytestring is to be
+    interpreted as an HTML document.)
+
+    3. An encoding detected through textual analysis by chardet,
+    cchardet, or a similar external library.
+
+    4. UTF-8.
+
+    5. Windows-1252.
+    """
+    def __init__(self, markup, override_encodings=None, is_html=False):
+        self.override_encodings = override_encodings or []
+        self.chardet_encoding = None
+        self.is_html = is_html
+        self.declared_encoding = None
+
+        # First order of business: strip a byte-order mark.
+        self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup)
+
+    def _usable(self, encoding, tried):
+        if encoding is not None:
+            encoding = encoding.lower()
+            if encoding not in tried:
+                tried.add(encoding)
+                return True
+        return False
+
+    @property
+    def encodings(self):
+        """Yield a number of encodings that might work for this markup."""
+        tried = set()
+        for e in self.override_encodings:
+            if self._usable(e, tried):
+                yield e
+
+        # Did the document originally start with a byte-order mark
+        # that indicated its encoding?
+        if self._usable(self.sniffed_encoding, tried):
+            yield self.sniffed_encoding
+
+        # Look within the document for an XML or HTML encoding
+        # declaration.
+        if self.declared_encoding is None:
+            self.declared_encoding = self.find_declared_encoding(
+                self.markup, self.is_html)
+        if self._usable(self.declared_encoding, tried):
+            yield self.declared_encoding
+
+        # Use third-party character set detection to guess at the
+        # encoding.
+        if self.chardet_encoding is None:
+            self.chardet_encoding = chardet_dammit(self.markup)
+        if self._usable(self.chardet_encoding, tried):
+            yield self.chardet_encoding
+
+        # As a last-ditch effort, try utf-8 and windows-1252.
+        for e in ('utf-8', 'windows-1252'):
+            if self._usable(e, tried):
+                yield e
+
+    @classmethod
+    def strip_byte_order_mark(cls, data):
+        """If a byte-order mark is present, strip it and return the encoding it implies."""
+        encoding = None
+        if (len(data) >= 4) and (data[:2] == b'\xfe\xff') \
+               and (data[2:4] != '\x00\x00'):
+            encoding = 'utf-16be'
+            data = data[2:]
+        elif (len(data) >= 4) and (data[:2] == b'\xff\xfe') \
+                 and (data[2:4] != '\x00\x00'):
+            encoding = 'utf-16le'
+            data = data[2:]
+        elif data[:3] == b'\xef\xbb\xbf':
+            encoding = 'utf-8'
+            data = data[3:]
+        elif data[:4] == b'\x00\x00\xfe\xff':
+            encoding = 'utf-32be'
+            data = data[4:]
+        elif data[:4] == b'\xff\xfe\x00\x00':
+            encoding = 'utf-32le'
+            data = data[4:]
+        return data, encoding
+
+    @classmethod
+    def find_declared_encoding(cls, markup, is_html=False, search_entire_document=False):
+        """Given a document, tries to find its declared encoding.
+
+        An XML encoding is declared at the beginning of the document.
+
+        An HTML encoding is declared in a <meta> tag, hopefully near the
+        beginning of the document.
+        """
+        if search_entire_document:
+            xml_endpos = html_endpos = len(markup)
+        else:
+            xml_endpos = 1024
+            html_endpos = max(2048, int(len(markup) * 0.05))
+            
+        declared_encoding = None
+        declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos)
+        if not declared_encoding_match and is_html:
+            declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos)
+        if declared_encoding_match is not None:
+            declared_encoding = declared_encoding_match.groups()[0].decode(
+                'ascii')
+        if declared_encoding:
+            return declared_encoding.lower()
+        return None
+
+class UnicodeDammit:
+    """A class for detecting the encoding of a *ML document and
+    converting it to a Unicode string. If the source encoding is
+    windows-1252, can replace MS smart quotes with their HTML or XML
+    equivalents."""
+
+    # This dictionary maps commonly seen values for "charset" in HTML
+    # meta tags to the corresponding Python codec names. It only covers
+    # values that aren't in Python's aliases and can't be determined
+    # by the heuristics in find_codec.
+    CHARSET_ALIASES = {"macintosh": "mac-roman",
+                       "x-sjis": "shift-jis"}
+
+    ENCODINGS_WITH_SMART_QUOTES = [
+        "windows-1252",
+        "iso-8859-1",
+        "iso-8859-2",
+        ]
+
+    def __init__(self, markup, override_encodings=[],
+                 smart_quotes_to=None, is_html=False):
+        self.smart_quotes_to = smart_quotes_to
+        self.tried_encodings = []
+        self.contains_replacement_characters = False
+        self.is_html = is_html
+
+        self.detector = EncodingDetector(markup, override_encodings, is_html)
+
+        # Short-circuit if the data is in Unicode to begin with.
+        if isinstance(markup, unicode) or markup == '':
+            self.markup = markup
+            self.unicode_markup = unicode(markup)
+            self.original_encoding = None
+            return
+
+        # The encoding detector may have stripped a byte-order mark.
+        # Use the stripped markup from this point on.
+        self.markup = self.detector.markup
+
+        u = None
+        for encoding in self.detector.encodings:
+            markup = self.detector.markup
+            u = self._convert_from(encoding)
+            if u is not None:
+                break
+
+        if not u:
+            # None of the encodings worked. As an absolute last resort,
+            # try them again with character replacement.
+
+            for encoding in self.detector.encodings:
+                if encoding != "ascii":
+                    u = self._convert_from(encoding, "replace")
+                if u is not None:
+                    logging.warning(
+                            "Some characters could not be decoded, and were "
+                            "replaced with REPLACEMENT CHARACTER.")
+                    self.contains_replacement_characters = True
+                    break
+
+        # If none of that worked, we could at this point force it to
+        # ASCII, but that would destroy so much data that I think
+        # giving up is better.
+        self.unicode_markup = u
+        if not u:
+            self.original_encoding = None
+
+    def _sub_ms_char(self, match):
+        """Changes a MS smart quote character to an XML or HTML
+        entity, or an ASCII character."""
+        orig = match.group(1)
+        if self.smart_quotes_to == 'ascii':
+            sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
+        else:
+            sub = self.MS_CHARS.get(orig)
+            if type(sub) == tuple:
+                if self.smart_quotes_to == 'xml':
+                    sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
+                else:
+                    sub = '&'.encode() + sub[0].encode() + ';'.encode()
+            else:
+                sub = sub.encode()
+        return sub
+
+    def _convert_from(self, proposed, errors="strict"):
+        proposed = self.find_codec(proposed)
+        if not proposed or (proposed, errors) in self.tried_encodings:
+            return None
+        self.tried_encodings.append((proposed, errors))
+        markup = self.markup
+        # Convert smart quotes to HTML if coming from an encoding
+        # that might have them.
+        if (self.smart_quotes_to is not None
+            and proposed in self.ENCODINGS_WITH_SMART_QUOTES):
+            smart_quotes_re = b"([\x80-\x9f])"
+            smart_quotes_compiled = re.compile(smart_quotes_re)
+            markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
+
+        try:
+            #print "Trying to convert document to %s (errors=%s)" % (
+            #    proposed, errors)
+            u = self._to_unicode(markup, proposed, errors)
+            self.markup = u
+            self.original_encoding = proposed
+        except Exception as e:
+            #print "That didn't work!"
+            #print e
+            return None
+        #print "Correct encoding: %s" % proposed
+        return self.markup
+
+    def _to_unicode(self, data, encoding, errors="strict"):
+        '''Given a string and its encoding, decodes the string into Unicode.
+        %encoding is a string recognized by encodings.aliases'''
+        return unicode(data, encoding, errors)
+
+    @property
+    def declared_html_encoding(self):
+        if not self.is_html:
+            return None
+        return self.detector.declared_encoding
+
+    def find_codec(self, charset):
+        value = (self._codec(self.CHARSET_ALIASES.get(charset, charset))
+               or (charset and self._codec(charset.replace("-", "")))
+               or (charset and self._codec(charset.replace("-", "_")))
+               or (charset and charset.lower())
+               or charset
+                )
+        if value:
+            return value.lower()
+        return None
+
+    def _codec(self, charset):
+        if not charset:
+            return charset
+        codec = None
+        try:
+            codecs.lookup(charset)
+            codec = charset
+        except (LookupError, ValueError):
+            pass
+        return codec
+
+
+    # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
+    MS_CHARS = {b'\x80': ('euro', '20AC'),
+                b'\x81': ' ',
+                b'\x82': ('sbquo', '201A'),
+                b'\x83': ('fnof', '192'),
+                b'\x84': ('bdquo', '201E'),
+                b'\x85': ('hellip', '2026'),
+                b'\x86': ('dagger', '2020'),
+                b'\x87': ('Dagger', '2021'),
+                b'\x88': ('circ', '2C6'),
+                b'\x89': ('permil', '2030'),
+                b'\x8A': ('Scaron', '160'),
+                b'\x8B': ('lsaquo', '2039'),
+                b'\x8C': ('OElig', '152'),
+                b'\x8D': '?',
+                b'\x8E': ('#x17D', '17D'),
+                b'\x8F': '?',
+                b'\x90': '?',
+                b'\x91': ('lsquo', '2018'),
+                b'\x92': ('rsquo', '2019'),
+                b'\x93': ('ldquo', '201C'),
+                b'\x94': ('rdquo', '201D'),
+                b'\x95': ('bull', '2022'),
+                b'\x96': ('ndash', '2013'),
+                b'\x97': ('mdash', '2014'),
+                b'\x98': ('tilde', '2DC'),
+                b'\x99': ('trade', '2122'),
+                b'\x9a': ('scaron', '161'),
+                b'\x9b': ('rsaquo', '203A'),
+                b'\x9c': ('oelig', '153'),
+                b'\x9d': '?',
+                b'\x9e': ('#x17E', '17E'),
+                b'\x9f': ('Yuml', ''),}
+
+    # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
+    # horrors like stripping diacritical marks to turn á into a, but also
+    # contains non-horrors like turning “ into ".
+    MS_CHARS_TO_ASCII = {
+        b'\x80' : 'EUR',
+        b'\x81' : ' ',
+        b'\x82' : ',',
+        b'\x83' : 'f',
+        b'\x84' : ',,',
+        b'\x85' : '...',
+        b'\x86' : '+',
+        b'\x87' : '++',
+        b'\x88' : '^',
+        b'\x89' : '%',
+        b'\x8a' : 'S',
+        b'\x8b' : '<',
+        b'\x8c' : 'OE',
+        b'\x8d' : '?',
+        b'\x8e' : 'Z',
+        b'\x8f' : '?',
+        b'\x90' : '?',
+        b'\x91' : "'",
+        b'\x92' : "'",
+        b'\x93' : '"',
+        b'\x94' : '"',
+        b'\x95' : '*',
+        b'\x96' : '-',
+        b'\x97' : '--',
+        b'\x98' : '~',
+        b'\x99' : '(TM)',
+        b'\x9a' : 's',
+        b'\x9b' : '>',
+        b'\x9c' : 'oe',
+        b'\x9d' : '?',
+        b'\x9e' : 'z',
+        b'\x9f' : 'Y',
+        b'\xa0' : ' ',
+        b'\xa1' : '!',
+        b'\xa2' : 'c',
+        b'\xa3' : 'GBP',
+        b'\xa4' : '$', #This approximation is especially parochial--this is the
+                       #generic currency symbol.
+        b'\xa5' : 'YEN',
+        b'\xa6' : '|',
+        b'\xa7' : 'S',
+        b'\xa8' : '..',
+        b'\xa9' : '',
+        b'\xaa' : '(th)',
+        b'\xab' : '<<',
+        b'\xac' : '!',
+        b'\xad' : ' ',
+        b'\xae' : '(R)',
+        b'\xaf' : '-',
+        b'\xb0' : 'o',
+        b'\xb1' : '+-',
+        b'\xb2' : '2',
+        b'\xb3' : '3',
+        b'\xb4' : ("'", 'acute'),
+        b'\xb5' : 'u',
+        b'\xb6' : 'P',
+        b'\xb7' : '*',
+        b'\xb8' : ',',
+        b'\xb9' : '1',
+        b'\xba' : '(th)',
+        b'\xbb' : '>>',
+        b'\xbc' : '1/4',
+        b'\xbd' : '1/2',
+        b'\xbe' : '3/4',
+        b'\xbf' : '?',
+        b'\xc0' : 'A',
+        b'\xc1' : 'A',
+        b'\xc2' : 'A',
+        b'\xc3' : 'A',
+        b'\xc4' : 'A',
+        b'\xc5' : 'A',
+        b'\xc6' : 'AE',
+        b'\xc7' : 'C',
+        b'\xc8' : 'E',
+        b'\xc9' : 'E',
+        b'\xca' : 'E',
+        b'\xcb' : 'E',
+        b'\xcc' : 'I',
+        b'\xcd' : 'I',
+        b'\xce' : 'I',
+        b'\xcf' : 'I',
+        b'\xd0' : 'D',
+        b'\xd1' : 'N',
+        b'\xd2' : 'O',
+        b'\xd3' : 'O',
+        b'\xd4' : 'O',
+        b'\xd5' : 'O',
+        b'\xd6' : 'O',
+        b'\xd7' : '*',
+        b'\xd8' : 'O',
+        b'\xd9' : 'U',
+        b'\xda' : 'U',
+        b'\xdb' : 'U',
+        b'\xdc' : 'U',
+        b'\xdd' : 'Y',
+        b'\xde' : 'b',
+        b'\xdf' : 'B',
+        b'\xe0' : 'a',
+        b'\xe1' : 'a',
+        b'\xe2' : 'a',
+        b'\xe3' : 'a',
+        b'\xe4' : 'a',
+        b'\xe5' : 'a',
+        b'\xe6' : 'ae',
+        b'\xe7' : 'c',
+        b'\xe8' : 'e',
+        b'\xe9' : 'e',
+        b'\xea' : 'e',
+        b'\xeb' : 'e',
+        b'\xec' : 'i',
+        b'\xed' : 'i',
+        b'\xee' : 'i',
+        b'\xef' : 'i',
+        b'\xf0' : 'o',
+        b'\xf1' : 'n',
+        b'\xf2' : 'o',
+        b'\xf3' : 'o',
+        b'\xf4' : 'o',
+        b'\xf5' : 'o',
+        b'\xf6' : 'o',
+        b'\xf7' : '/',
+        b'\xf8' : 'o',
+        b'\xf9' : 'u',
+        b'\xfa' : 'u',
+        b'\xfb' : 'u',
+        b'\xfc' : 'u',
+        b'\xfd' : 'y',
+        b'\xfe' : 'b',
+        b'\xff' : 'y',
+        }
+
+    # A map used when removing rogue Windows-1252/ISO-8859-1
+    # characters in otherwise UTF-8 documents.
+    #
+    # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
+    # Windows-1252.
+    WINDOWS_1252_TO_UTF8 = {
+        0x80 : b'\xe2\x82\xac', # €
+        0x82 : b'\xe2\x80\x9a', # ‚
+        0x83 : b'\xc6\x92',     # ƒ
+        0x84 : b'\xe2\x80\x9e', # „
+        0x85 : b'\xe2\x80\xa6', # …
+        0x86 : b'\xe2\x80\xa0', # †
+        0x87 : b'\xe2\x80\xa1', # ‡
+        0x88 : b'\xcb\x86',     # ˆ
+        0x89 : b'\xe2\x80\xb0', # ‰
+        0x8a : b'\xc5\xa0',     # Š
+        0x8b : b'\xe2\x80\xb9', # ‹
+        0x8c : b'\xc5\x92',     # Œ
+        0x8e : b'\xc5\xbd',     # Ž
+        0x91 : b'\xe2\x80\x98', # ‘
+        0x92 : b'\xe2\x80\x99', # ’
+        0x93 : b'\xe2\x80\x9c', # “
+        0x94 : b'\xe2\x80\x9d', # ”
+        0x95 : b'\xe2\x80\xa2', # •
+        0x96 : b'\xe2\x80\x93', # –
+        0x97 : b'\xe2\x80\x94', # —
+        0x98 : b'\xcb\x9c',     # ˜
+        0x99 : b'\xe2\x84\xa2', # ™
+        0x9a : b'\xc5\xa1',     # š
+        0x9b : b'\xe2\x80\xba', # ›
+        0x9c : b'\xc5\x93',     # œ
+        0x9e : b'\xc5\xbe',     # ž
+        0x9f : b'\xc5\xb8',     # Ÿ
+        0xa0 : b'\xc2\xa0',     #  
+        0xa1 : b'\xc2\xa1',     # ¡
+        0xa2 : b'\xc2\xa2',     # ¢
+        0xa3 : b'\xc2\xa3',     # £
+        0xa4 : b'\xc2\xa4',     # ¤
+        0xa5 : b'\xc2\xa5',     # ¥
+        0xa6 : b'\xc2\xa6',     # ¦
+        0xa7 : b'\xc2\xa7',     # §
+        0xa8 : b'\xc2\xa8',     # ¨
+        0xa9 : b'\xc2\xa9',     # ©
+        0xaa : b'\xc2\xaa',     # ª
+        0xab : b'\xc2\xab',     # «
+        0xac : b'\xc2\xac',     # ¬
+        0xad : b'\xc2\xad',     # ­
+        0xae : b'\xc2\xae',     # ®
+        0xaf : b'\xc2\xaf',     # ¯
+        0xb0 : b'\xc2\xb0',     # °
+        0xb1 : b'\xc2\xb1',     # ±
+        0xb2 : b'\xc2\xb2',     # ²
+        0xb3 : b'\xc2\xb3',     # ³
+        0xb4 : b'\xc2\xb4',     # ´
+        0xb5 : b'\xc2\xb5',     # µ
+        0xb6 : b'\xc2\xb6',     # ¶
+        0xb7 : b'\xc2\xb7',     # ·
+        0xb8 : b'\xc2\xb8',     # ¸
+        0xb9 : b'\xc2\xb9',     # ¹
+        0xba : b'\xc2\xba',     # º
+        0xbb : b'\xc2\xbb',     # »
+        0xbc : b'\xc2\xbc',     # ¼
+        0xbd : b'\xc2\xbd',     # ½
+        0xbe : b'\xc2\xbe',     # ¾
+        0xbf : b'\xc2\xbf',     # ¿
+        0xc0 : b'\xc3\x80',     # À
+        0xc1 : b'\xc3\x81',     # Á
+        0xc2 : b'\xc3\x82',     # Â
+        0xc3 : b'\xc3\x83',     # Ã
+        0xc4 : b'\xc3\x84',     # Ä
+        0xc5 : b'\xc3\x85',     # Å
+        0xc6 : b'\xc3\x86',     # Æ
+        0xc7 : b'\xc3\x87',     # Ç
+        0xc8 : b'\xc3\x88',     # È
+        0xc9 : b'\xc3\x89',     # É
+        0xca : b'\xc3\x8a',     # Ê
+        0xcb : b'\xc3\x8b',     # Ë
+        0xcc : b'\xc3\x8c',     # Ì
+        0xcd : b'\xc3\x8d',     # Í
+        0xce : b'\xc3\x8e',     # Î
+        0xcf : b'\xc3\x8f',     # Ï
+        0xd0 : b'\xc3\x90',     # Ð
+        0xd1 : b'\xc3\x91',     # Ñ
+        0xd2 : b'\xc3\x92',     # Ò
+        0xd3 : b'\xc3\x93',     # Ó
+        0xd4 : b'\xc3\x94',     # Ô
+        0xd5 : b'\xc3\x95',     # Õ
+        0xd6 : b'\xc3\x96',     # Ö
+        0xd7 : b'\xc3\x97',     # ×
+        0xd8 : b'\xc3\x98',     # Ø
+        0xd9 : b'\xc3\x99',     # Ù
+        0xda : b'\xc3\x9a',     # Ú
+        0xdb : b'\xc3\x9b',     # Û
+        0xdc : b'\xc3\x9c',     # Ü
+        0xdd : b'\xc3\x9d',     # Ý
+        0xde : b'\xc3\x9e',     # Þ
+        0xdf : b'\xc3\x9f',     # ß
+        0xe0 : b'\xc3\xa0',     # à
+        0xe1 : b'\xa1',     # á
+        0xe2 : b'\xc3\xa2',     # â
+        0xe3 : b'\xc3\xa3',     # ã
+        0xe4 : b'\xc3\xa4',     # ä
+        0xe5 : b'\xc3\xa5',     # å
+        0xe6 : b'\xc3\xa6',     # æ
+        0xe7 : b'\xc3\xa7',     # ç
+        0xe8 : b'\xc3\xa8',     # è
+        0xe9 : b'\xc3\xa9',     # é
+        0xea : b'\xc3\xaa',     # ê
+        0xeb : b'\xc3\xab',     # ë
+        0xec : b'\xc3\xac',     # ì
+        0xed : b'\xc3\xad',     # í
+        0xee : b'\xc3\xae',     # î
+        0xef : b'\xc3\xaf',     # ï
+        0xf0 : b'\xc3\xb0',     # ð
+        0xf1 : b'\xc3\xb1',     # ñ
+        0xf2 : b'\xc3\xb2',     # ò
+        0xf3 : b'\xc3\xb3',     # ó
+        0xf4 : b'\xc3\xb4',     # ô
+        0xf5 : b'\xc3\xb5',     # õ
+        0xf6 : b'\xc3\xb6',     # ö
+        0xf7 : b'\xc3\xb7',     # ÷
+        0xf8 : b'\xc3\xb8',     # ø
+        0xf9 : b'\xc3\xb9',     # ù
+        0xfa : b'\xc3\xba',     # ú
+        0xfb : b'\xc3\xbb',     # û
+        0xfc : b'\xc3\xbc',     # ü
+        0xfd : b'\xc3\xbd',     # ý
+        0xfe : b'\xc3\xbe',     # þ
+        }
+
+    MULTIBYTE_MARKERS_AND_SIZES = [
+        (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
+        (0xe0, 0xef, 3), # 3-byte characters start with E0-EF
+        (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
+        ]
+
+    FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
+    LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
+
+    @classmethod
+    def detwingle(cls, in_bytes, main_encoding="utf8",
+                  embedded_encoding="windows-1252"):
+        """Fix characters from one encoding embedded in some other encoding.
+
+        Currently the only situation supported is Windows-1252 (or its
+        subset ISO-8859-1), embedded in UTF-8.
+
+        The input must be a bytestring. If you've already converted
+        the document to Unicode, you're too late.
+
+        The output is a bytestring in which `embedded_encoding`
+        characters have been converted to their `main_encoding`
+        equivalents.
+        """
+        if embedded_encoding.replace('_', '-').lower() not in (
+            'windows-1252', 'windows_1252'):
+            raise NotImplementedError(
+                "Windows-1252 and ISO-8859-1 are the only currently supported "
+                "embedded encodings.")
+
+        if main_encoding.lower() not in ('utf8', 'utf-8'):
+            raise NotImplementedError(
+                "UTF-8 is the only currently supported main encoding.")
+
+        byte_chunks = []
+
+        chunk_start = 0
+        pos = 0
+        while pos < len(in_bytes):
+            byte = in_bytes[pos]
+            if not isinstance(byte, int):
+                # Python 2.x
+                byte = ord(byte)
+            if (byte >= cls.FIRST_MULTIBYTE_MARKER
+                and byte <= cls.LAST_MULTIBYTE_MARKER):
+                # This is the start of a UTF-8 multibyte character. Skip
+                # to the end.
+                for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
+                    if byte >= start and byte <= end:
+                        pos += size
+                        break
+            elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
+                # We found a Windows-1252 character!
+                # Save the string up to this point as a chunk.
+                byte_chunks.append(in_bytes[chunk_start:pos])
+
+                # Now translate the Windows-1252 character into UTF-8
+                # and add it as another, one-byte chunk.
+                byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
+                pos += 1
+                chunk_start = pos
+            else:
+                # Go on to the next character.
+                pos += 1
+        if chunk_start == 0:
+            # The string is unchanged.
+            return in_bytes
+        else:
+            # Store the final chunk.
+            byte_chunks.append(in_bytes[chunk_start:])
+        return b''.join(byte_chunks)
+

bs4/diagnose.py

@@ -0,0 +1,204 @@
+"""Diagnostic functions, mainly for use when doing tech support."""
+import cProfile
+from StringIO import StringIO
+from HTMLParser import HTMLParser
+import bs4
+from bs4 import BeautifulSoup, __version__
+from bs4.builder import builder_registry
+
+import os
+import pstats
+import random
+import tempfile
+import time
+import traceback
+import sys
+import cProfile
+
+def diagnose(data):
+    """Diagnostic suite for isolating common problems."""
+    print "Diagnostic running on Beautiful Soup %s" % __version__
+    print "Python version %s" % sys.version
+
+    basic_parsers = ["html.parser", "html5lib", "lxml"]
+    for name in basic_parsers:
+        for builder in builder_registry.builders:
+            if name in builder.features:
+                break
+        else:
+            basic_parsers.remove(name)
+            print (
+                "I noticed that %s is not installed. Installing it may help." %
+                name)
+
+    if 'lxml' in basic_parsers:
+        basic_parsers.append(["lxml", "xml"])
+        from lxml import etree
+        print "Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))
+
+    if 'html5lib' in basic_parsers:
+        import html5lib
+        print "Found html5lib version %s" % html5lib.__version__
+
+    if hasattr(data, 'read'):
+        data = data.read()
+    elif os.path.exists(data):
+        print '"%s" looks like a filename. Reading data from the file.' % data
+        data = open(data).read()
+    elif data.startswith("http:") or data.startswith("https:"):
+        print '"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data
+        print "You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup."
+        return
+    print
+
+    for parser in basic_parsers:
+        print "Trying to parse your markup with %s" % parser
+        success = False
+        try:
+            soup = BeautifulSoup(data, parser)
+            success = True
+        except Exception, e:
+            print "%s could not parse the markup." % parser
+            traceback.print_exc()
+        if success:
+            print "Here's what %s did with the markup:" % parser
+            print soup.prettify()
+
+        print "-" * 80
+
+def lxml_trace(data, html=True, **kwargs):
+    """Print out the lxml events that occur during parsing.
+
+    This lets you see how lxml parses a document when no Beautiful
+    Soup code is running.
+    """
+    from lxml import etree
+    for event, element in etree.iterparse(StringIO(data), html=html, **kwargs):
+        print("%s, %4s, %s" % (event, element.tag, element.text))
+
+class AnnouncingParser(HTMLParser):
+    """Announces HTMLParser parse events, without doing anything else."""
+
+    def _p(self, s):
+        print(s)
+
+    def handle_starttag(self, name, attrs):
+        self._p("%s START" % name)
+
+    def handle_endtag(self, name):
+        self._p("%s END" % name)
+
+    def handle_data(self, data):
+        self._p("%s DATA" % data)
+
+    def handle_charref(self, name):
+        self._p("%s CHARREF" % name)
+
+    def handle_entityref(self, name):
+        self._p("%s ENTITYREF" % name)
+
+    def handle_comment(self, data):
+        self._p("%s COMMENT" % data)
+
+    def handle_decl(self, data):
+        self._p("%s DECL" % data)
+
+    def unknown_decl(self, data):
+        self._p("%s UNKNOWN-DECL" % data)
+
+    def handle_pi(self, data):
+        self._p("%s PI" % data)
+
+def htmlparser_trace(data):
+    """Print out the HTMLParser events that occur during parsing.
+
+    This lets you see how HTMLParser parses a document when no
+    Beautiful Soup code is running.
+    """
+    parser = AnnouncingParser()
+    parser.feed(data)
+
+_vowels = "aeiou"
+_consonants = "bcdfghjklmnpqrstvwxyz"
+
+def rword(length=5):
+    "Generate a random word-like string."
+    s = ''
+    for i in range(length):
+        if i % 2 == 0:
+            t = _consonants
+        else:
+            t = _vowels
+        s += random.choice(t)
+    return s
+
+def rsentence(length=4):
+    "Generate a random sentence-like string."
+    return " ".join(rword(random.randint(4,9)) for i in range(length))
+        
+def rdoc(num_elements=1000):
+    """Randomly generate an invalid HTML document."""
+    tag_names = ['p', 'div', 'span', 'i', 'b', 'script', 'table']
+    elements = []
+    for i in range(num_elements):
+        choice = random.randint(0,3)
+        if choice == 0:
+            # New tag.
+            tag_name = random.choice(tag_names)
+            elements.append("<%s>" % tag_name)
+        elif choice == 1:
+            elements.append(rsentence(random.randint(1,4)))
+        elif choice == 2:
+            # Close a tag.
+            tag_name = random.choice(tag_names)
+            elements.append("</%s>" % tag_name)
+    return "<html>" + "\n".join(elements) + "</html>"
+
+def benchmark_parsers(num_elements=100000):
+    """Very basic head-to-head performance benchmark."""
+    print "Comparative parser benchmark on Beautiful Soup %s" % __version__
+    data = rdoc(num_elements)
+    print "Generated a large invalid HTML document (%d bytes)." % len(data)
+    
+    for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
+        success = False
+        try:
+            a = time.time()
+            soup = BeautifulSoup(data, parser)
+            b = time.time()
+            success = True
+        except Exception, e:
+            print "%s could not parse the markup." % parser
+            traceback.print_exc()
+        if success:
+            print "BS4+%s parsed the markup in %.2fs." % (parser, b-a)
+
+    from lxml import etree
+    a = time.time()
+    etree.HTML(data)
+    b = time.time()
+    print "Raw lxml parsed the markup in %.2fs." % (b-a)
+
+    import html5lib
+    parser = html5lib.HTMLParser()
+    a = time.time()
+    parser.parse(data)
+    b = time.time()
+    print "Raw html5lib parsed the markup in %.2fs." % (b-a)
+
+def profile(num_elements=100000, parser="lxml"):
+
+    filehandle = tempfile.NamedTemporaryFile()
+    filename = filehandle.name
+
+    data = rdoc(num_elements)
+    vars = dict(bs4=bs4, data=data, parser=parser)
+    cProfile.runctx('bs4.BeautifulSoup(data, parser)' , vars, vars, filename)
+
+    stats = pstats.Stats(filename)
+    # stats.strip_dirs()
+    stats.sort_stats("cumulative")
+    stats.print_stats('_html5lib|bs4', 50)
+
+if __name__ == '__main__':
+    diagnose(sys.stdin.read())

calibre-plugin/__init__.py

@@ -4,9 +4,25 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2013, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import sys
+if sys.version_info >= (2, 7):
+    import logging
+    logger = logging.getLogger(__name__)
+    loghandler=logging.StreamHandler()
+    loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+    logger.addHandler(loghandler)
+    loghandler.setLevel(logging.DEBUG)
+    logger.setLevel(logging.DEBUG)
+
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
@@ -23,11 +39,11 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     calibre utilities to run without needing to load the GUI libraries.
     '''
     name                = 'FanFictionDownLoader'
-    description         = 'UI plugin to download FanFiction stories from various sites.'
+    description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 41)
-    minimum_calibre_version = (0, 8, 57)
+    version             = (2, 1, 5)
+    minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name
@@ -79,29 +95,37 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
         if ac is not None:
             ac.apply_settings()
 
+    def load_actual_plugin(self, gui):
+        with self: # so the sys.path was modified while loading the
+                   # plug impl.
+            return InterfaceActionBase.load_actual_plugin(self,gui)
+
     def cli_main(self,argv):
-        # I believe there's no performance hit loading these here when
-        # CLI--it would load everytime anyway.
-        from StringIO import StringIO
-        from calibre.library import db
-        from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
-        from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
-        from calibre.utils.config import prefs as calibre_prefs
-        from optparse import OptionParser      
 
-        parser = OptionParser('%prog --run-plugin '+self.name+' -- [options] <storyurl>')
-        parser.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
-        # parser.add_option('--dont-notify-gui', default=False, action='store_true',
-        #               help=_('Do not notify the running calibre GUI (if any) that the database has'
-        #                      ' changed. Use with care, as it can lead to database corruption!'))
+        with self: # so the sys.path was modified appropriately
+            # I believe there's no performance hit loading these here when
+            # CLI--it would load everytime anyway.
+            from StringIO import StringIO
+            from calibre.library import db
+            from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
+            from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
+            from calibre.utils.config import prefs as calibre_prefs
+            from optparse import OptionParser      
+    
+            parser = OptionParser('%prog --run-plugin '+self.name+' -- [options] <storyurl>')
+            parser.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
+            # parser.add_option('--dont-notify-gui', default=False, action='store_true',
+            #               help=_('Do not notify the running calibre GUI (if any) that the database has'
+            #                      ' changed. Use with care, as it can lead to database corruption!'))
+    
+            pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
+                     or not x.startswith('-')]
+            opts, args = parser.parse_args(pargs)
+    
+            ffdl_prefs = PrefsFacade(db(path=opts.library_path,
+                                        read_only=True))
 
-        pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
-                 or not x.startswith('-')]
-        opts, args = parser.parse_args(pargs)
-
-        ffdl_prefs = PrefsFacade(db(path=opts.library_path,
-                                    read_only=True))
-        ffdl_main(argv[1:],
-                  parser=parser,
-                  passed_defaultsini=StringIO(get_resources("defaults.ini")),
-                  passed_personalini=StringIO(ffdl_prefs["personal.ini"]))
+            ffdl_main(argv[1:],
+                      parser=parser,
+                      passed_defaultsini=StringIO(get_resources("defaults.ini")),
+                      passed_personalini=StringIO(ffdl_prefs["personal.ini"]))

calibre-plugin/basicinihighlighter.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2014, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import re
+
+try:
+    from PyQt5.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush)
+except ImportError as e:
+    from PyQt4.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush)
+
+class BasicIniHighlighter(QSyntaxHighlighter):
+    '''
+    QSyntaxHighlighter class for use with QTextEdit for highlighting
+    ini config files.
+
+    I looked high and low to find a high lighter for basic ini config
+    format, so I'm leaving this in the project even though I'm not
+    using.
+    '''
+    
+    def __init__( self, parent, theme ):
+        QSyntaxHighlighter.__init__( self, parent )
+        self.parent = parent
+        
+        self.highlightingRules = []
+
+        # keyword
+        self.highlightingRules.append( HighlightingRule( r"^[^:=\s][^:=]*[:=]",
+                                                         Qt.blue,
+                                                         Qt.SolidPattern ) )
+
+        # section
+        self.highlightingRules.append( HighlightingRule( r"^\[[^\]]+\]",
+                                                         Qt.darkBlue,
+                                                         Qt.SolidPattern ) )
+
+        # comment
+        self.highlightingRules.append( HighlightingRule( r"#[^\n]*" ,
+                                                         Qt.darkYellow,
+                                                         Qt.SolidPattern ) )
+
+    def highlightBlock( self, text ):
+        for rule in self.highlightingRules:
+            for match in rule.pattern.finditer(text):
+                self.setFormat( match.start(), match.end()-match.start(), rule.highlight )
+        self.setCurrentBlockState( 0 )
+
+class HighlightingRule():
+    def __init__( self, pattern, color, style ):
+        if isinstance(pattern,basestring):
+            self.pattern = re.compile(pattern)
+        else:
+            self.pattern=pattern
+        charfmt = QTextCharFormat()
+        brush = QBrush(color, style)
+        charfmt.setForeground(brush)
+        self.highlight = charfmt
+    

calibre-plugin/common_utils.py

@@ -8,12 +8,19 @@ __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
-from PyQt4 import QtGui
-from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                      QTextEdit,
-                      QListWidget, QAbstractItemView)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+    
 from calibre.constants import iswindows
 from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name

calibre-plugin/dialogs.py

@@ -4,25 +4,62 @@ from __future__ import (unicode_literals, division,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2015, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import traceback, re
 from functools import partial
 
+import logging
+logger = logging.getLogger(__name__)
+
 import urllib
 import email
 
-from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
-                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
-                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
-                      QGroupBox, QFrame)
+from datetime import datetime
+
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5 import QtCore
+    from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QFont, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame, QTextBrowser, QSize, QAction)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4 import QtCore
+    from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QFont, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame, QTextBrowser, QSize, QAction)
+
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
@@ -30,13 +67,19 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
 
-SKIP=u'Skip'
-ADDNEW=u'Add New Book'
-UPDATE=u'Update EPUB if New Chapters'
-UPDATEALWAYS=u'Update EPUB Always'
-OVERWRITE=u'Overwrite if Newer'
-OVERWRITEALWAYS=u'Overwrite Always'
-CALIBREONLY=u'Update Calibre Metadata Only'
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable \
+    import (get_valid_sections, get_valid_entries,
+            get_valid_keywords, get_valid_entry_keywords)
+
+from inihighlighter import IniHighlighter
+
+SKIP=_('Skip')
+ADDNEW=_('Add New Book')
+UPDATE=_('Update EPUB if New Chapters')
+UPDATEALWAYS=_('Update EPUB Always')
+OVERWRITE=_('Overwrite if Newer')
+OVERWRITEALWAYS=_('Overwrite Always')
+CALIBREONLY=_('Update Calibre Metadata Only')
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
@@ -45,6 +88,32 @@ collision_order=[SKIP,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
 
+# best idea I've had for how to deal with config/pref saving the
+# collision name in english.
+SAVE_SKIP='Skip'
+SAVE_ADDNEW='Add New Book'
+SAVE_UPDATE='Update EPUB if New Chapters'
+SAVE_UPDATEALWAYS='Update EPUB Always'
+SAVE_OVERWRITE='Overwrite if Newer'
+SAVE_OVERWRITEALWAYS='Overwrite Always'
+SAVE_CALIBREONLY='Update Calibre Metadata Only'
+save_collisions={
+    SKIP:SAVE_SKIP,
+    ADDNEW:SAVE_ADDNEW,
+    UPDATE:SAVE_UPDATE,
+    UPDATEALWAYS:SAVE_UPDATEALWAYS,
+    OVERWRITE:SAVE_OVERWRITE,
+    OVERWRITEALWAYS:SAVE_OVERWRITEALWAYS,
+    CALIBREONLY:SAVE_CALIBREONLY,
+    SAVE_SKIP:SKIP,
+    SAVE_ADDNEW:ADDNEW,
+    SAVE_UPDATE:UPDATE,
+    SAVE_UPDATEALWAYS:UPDATEALWAYS,
+    SAVE_OVERWRITE:OVERWRITE,
+    SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
+    SAVE_CALIBREONLY:CALIBREONLY,
+    }
+    
 anthology_collision_order=[UPDATE,
                            UPDATEALWAYS,
                            OVERWRITEALWAYS]
@@ -98,6 +167,7 @@ class RejectUrlEntry:
     def fullnote(self):
         retval = ""
         if self.title and self.auth:
+            # don't translate--ends up being saved and confuses regex above.
             retval = retval + "%s by %s"%(self.title,self.auth)
             if self.note:
                 retval = retval + " - "
@@ -107,19 +177,6 @@ class RejectUrlEntry:
             
         return retval
 
-# This is a more than slightly kludgey way to get
-# EditWithComplete to *not* alpha-order the reasons, but leave
-# them in the order entered.  If
-# calibre.gui2.complete2.CompleteModel.set_items ever changes,
-# this function will need to also.
-def complete_model_set_items_kludge(self, items):
-    items = [unicode(x.strip()) for x in items]
-    items = [x for x in items if x]
-    items = tuple(items)
-    self.all_items = self.current_items = items
-    self.current_prefix = ''
-    self.reset()
-    
 class NotGoingToDownload(Exception):
     def __init__(self,error,icon='dialog_error.png'):
         self.error=error
@@ -157,9 +214,9 @@ class DroppableQTextEdit(QTextEdit):
                             urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 else:
                     urllist.extend(get_urls_from_text("%s"%msg))
-        
         if urllist:
             self.append("\n".join(urllist))
+            return None
         return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
@@ -186,7 +243,7 @@ class AddNewDialog(SizePersistedDialog):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        self.setWindowTitle('FanFictionDownLoader')
+        self.setWindowTitle(_('FanFictionDownLoader'))
         self.setWindowIcon(icon)
 
         self.toplabel=QLabel("Toplabel")
@@ -203,11 +260,11 @@ class AddNewDialog(SizePersistedDialog):
         # elements to show again when doing *update* merge
         self.mergeupdateshow = []
 
-        self.groupbox = QGroupBox("Show Download Options")
+        self.groupbox = QGroupBox(_("Show Download Options"))
         self.groupbox.setCheckable(True)
         self.groupbox.setChecked(False)
         self.groupbox.setFlat(True)
-        print("style:%s"%self.groupbox.styleSheet())
+        #print("style:%s"%self.groupbox.styleSheet())
         self.groupbox.setStyleSheet(gpstyle)
 
         self.gbf = QFrame()
@@ -222,7 +279,7 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.toggled.connect(self.gbf.setVisible)
         
         horz = QHBoxLayout()
-        label = QLabel('Output &Format:')
+        label = QLabel(_('Output &Format:'))
         self.mergehide.append(label)
         
         self.fileform = QComboBox(self)
@@ -230,7 +287,7 @@ class AddNewDialog(SizePersistedDialog):
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         
         horz.addWidget(label)
@@ -246,7 +303,7 @@ class AddNewDialog(SizePersistedDialog):
         self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
@@ -258,15 +315,15 @@ class AddNewDialog(SizePersistedDialog):
         self.mergeupdateshow.append(self.collision)
 
         horz = QHBoxLayout()
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
+        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
@@ -313,10 +370,10 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.setVisible(not(self.merge and self.newmerge))
         
         if self.merge:
-            self.toplabel.setText('Story URL(s) for anthology, one per line:')
-            self.url.setToolTip('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.')
-            self.collisionlabel.setText('If Story Already Exists in Anthology?')
-            self.collision.setToolTip("What to do if there's already an existing story with the same URL in the anthology.")
+            self.toplabel.setText(_('Story URL(s) for anthology, one per line:'))
+            self.url.setToolTip(_('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'))
+            self.collisionlabel.setText(_('If Story Already Exists in Anthology?'))
+            self.collision.setToolTip(_("What to do if there's already an existing story with the same URL in the anthology."))
             for widget in self.mergehide:
                 widget.setVisible(False)
             if not self.newmerge:
@@ -325,10 +382,10 @@ class AddNewDialog(SizePersistedDialog):
         else:
             for widget in self.mergehide:
                 widget.setVisible(True)
-            self.toplabel.setText('Story URL(s), one per line:')
-            self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
-            self.collisionlabel.setText('If Story Already Exists?')
-            self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+            self.toplabel.setText(_('Story URL(s), one per line:'))
+            self.url.setToolTip(_('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.'))
+            self.collisionlabel.setText(_('If Story Already Exists?'))
+            self.collision.setToolTip(_("What to do if there's already an existing story with the same URL or title and author."))
 
         # Need to re-able after hiding/showing
         self.setAcceptDrops(True)
@@ -344,10 +401,10 @@ class AddNewDialog(SizePersistedDialog):
 
         # add collision options
         self.set_collisions()
-        
-        i = self.collision.findText(self.prefs['collision'])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
+            
         self.updatemeta.setChecked(self.prefs['updatemeta'])
             
         if not self.merge:
@@ -428,18 +485,18 @@ class CollectURLDialog(SizePersistedDialog):
         self.url.setText(url_text)
         self.l.addWidget(self.url,1,1,1,2)
    
-        self.indiv_button = QPushButton('For Individual Books', self)
-        self.indiv_button.setToolTip('Get URLs and go to dialog for individual story downloads.')
+        self.indiv_button = QPushButton(_('For Individual Books'), self)
+        self.indiv_button.setToolTip(_('Get URLs and go to dialog for individual story downloads.'))
         self.indiv_button.clicked.connect(self.indiv)
         self.l.addWidget(self.indiv_button,2,0)
 
-        self.merge_button = QPushButton('For Anthology Epub', self)
-        self.merge_button.setToolTip('Get URLs and go to dialog for Anthology download.\nRequires EpubMerge 1.3.1+ plugin.')
+        self.merge_button = QPushButton(_('For Anthology Epub'), self)
+        self.merge_button.setToolTip(_('Get URLs and go to dialog for Anthology download.\nRequires %s plugin.')%'EpubMerge 1.3.1+')
         self.merge_button.clicked.connect(self.merge)
         self.l.addWidget(self.merge_button,2,1)
         self.merge_button.setEnabled(epubmerge_plugin!=None)
 
-        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,2,2)
 
@@ -470,30 +527,30 @@ class UserPassDialog(QDialog):
         self.l = QGridLayout()
         self.setLayout(self.l)
 
-        if exception.passwdonly:
-            self.setWindowTitle('Password')
-            self.l.addWidget(QLabel("Author requires a password for this story(%s)."%exception.url),0,0,1,2)
+        if exception and exception.passwdonly:
+            self.setWindowTitle(_('Password'))
+            self.l.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
             # user isn't used, but it's easier to still have it for
             # post processing.
             self.user = FakeLineEdit()
         else:
-            self.setWindowTitle('User/Password')
-            self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
+            self.setWindowTitle(_('User/Password'))
+            self.l.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
         
-            self.l.addWidget(QLabel("User:"),1,0)
+            self.l.addWidget(QLabel(_("User:")),1,0)
             self.user = QLineEdit(self)
             self.l.addWidget(self.user,1,1)
    
-        self.l.addWidget(QLabel("Password:"),2,0)
+        self.l.addWidget(QLabel(_("Password:")),2,0)
         self.passwd = QLineEdit(self)
         self.passwd.setEchoMode(QLineEdit.Password)
         self.l.addWidget(self.passwd,2,1)
    
-        self.ok_button = QPushButton('OK', self)
+        self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.ok)
         self.l.addWidget(self.ok_button,3,0)
 
-        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,3,1)
 
@@ -515,12 +572,12 @@ class LoopProgressDialog(QProgressDialog):
                  book_list,
                  foreach_function,
                  finish_function,
-                 init_label="Fetching metadata for stories...",
-                 win_title="Downloading metadata for stories",
-                 status_prefix="Fetched metadata for"):
+                 init_label=_("Fetching metadata for stories..."),
+                 win_title=_("Downloading metadata for stories"),
+                 status_prefix=_("Fetched metadata for")):
         QProgressDialog.__init__(self,
                                  init_label,
-                                 QString(), 0, len(book_list), gui)
+                                 _('Cancel'), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.book_list = book_list
@@ -528,16 +585,27 @@ class LoopProgressDialog(QProgressDialog):
         self.finish_function = finish_function
         self.status_prefix = status_prefix
         self.i = 0
-        
+        self.start_time = datetime.now()
+
+        # can't import at file load.
+        from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs
+        self.show_est_time = prefs['show_est_time']
+
         ## self.do_loop does QTimer.singleShot on self.do_loop also.
         ## A weird way to do a loop, but that was the example I had.
         QTimer.singleShot(0, self.do_loop)
         self.exec_()
 
     def updateStatus(self):
-        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
+        remaining_time_string = ''
+        if self.show_est_time and self.i > -1:
+            time_spent = (datetime.now() - self.start_time).total_seconds()
+            estimated_remaining = (time_spent/(self.i+1)) * len(self.book_list) - time_spent
+            remaining_time_string = _(' - %s estimated until done') % ( time_duration_format(estimated_remaining))
+
+        self.setLabelText('%s %d / %d%s' % (self.status_prefix, self.i+1, len(self.book_list), remaining_time_string))
         self.setValue(self.i+1)
-        print(self.labelText())
+        #print(self.labelText())
 
     def do_loop(self):
 
@@ -558,7 +626,7 @@ class LoopProgressDialog(QProgressDialog):
         except Exception as e:
             book['good']=False
             book['comment']=unicode(e)
-            print("Exception: %s:%s"%(book,unicode(e)))
+            logger.error("Exception: %s:%s"%(book,unicode(e)))
             traceback.print_exc()
             
         self.updateStatus()
@@ -574,6 +642,37 @@ class LoopProgressDialog(QProgressDialog):
         # Queues a job to process these books in the background.
         self.finish_function(self.book_list)
 
+def time_duration_format(seconds):
+    """
+    Convert seconds into a string describing the duration in larger time units (seconds, minutes, hours, days)
+    Only returns the two largest time divisions (eg, will drop seconds if there's hours remaining)
+
+    :param seconds: number of seconds
+    :return: string description of the duration
+    """
+    periods = [
+        (_('%d day'),_('%d days'),       60*60*24),
+        (_('%d hour'),_('%d hours'),     60*60),
+        (_('%d minute'),_('%d minutes'), 60),
+        (_('%d second'),_('%d seconds'), 1)
+        ]
+
+    strings = []
+    for period_label, period_plural_label, period_seconds in periods:
+        if seconds > period_seconds:
+            period_value, seconds = divmod(seconds,period_seconds)
+            if period_value == 1:
+                strings.append( period_label % period_value)
+            else:
+                strings.append(period_plural_label % period_value)
+            if len(strings) == 2:
+                break
+
+    if len(strings) == 0:
+        return _('less than 1 second')
+    else:
+        return ', '.join(strings)
+
 class AboutDialog(QDialog):
 
     def __init__(self, parent, icon, text):
@@ -641,7 +740,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
         self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected books from the list')
+        self.remove_button.setToolTip(_('Remove selected books from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
@@ -650,7 +749,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         options_layout = QHBoxLayout()
 
-        groupbox = QGroupBox("Show Download Options")
+        groupbox = QGroupBox(_("Show Download Options"))
         groupbox.setCheckable(True)
         groupbox.setChecked(False)
         groupbox.setFlat(True)
@@ -667,7 +766,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         gbf.setVisible(False)
         groupbox.toggled.connect(gbf.setVisible)
         
-        label = QLabel('Output &Format:')
+        label = QLabel(_('Output &Format:'))
         gbl.addWidget(label)
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
@@ -675,30 +774,30 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
         self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
         gbl.addWidget(self.fileform)
         
-        label = QLabel('Update Mode:')
+        label = QLabel(_('Update Mode:'))
         gbl.addWidget(label)
         self.collision = QComboBox(self)
-        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
+        self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
         gbl.addWidget(self.collision)
 
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
                 
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
+        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
@@ -751,7 +850,7 @@ class StoryListTableWidget(QTableWidget):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(books))
-        header_labels = ['','Title', 'Author', 'URL', 'Comment']
+        header_labels = ['',_('Title'), _('Author'), 'URL', _('Comment')]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
@@ -790,11 +889,11 @@ class StoryListTableWidget(QTableWidget):
             icon = get_icon(book['icon'])
 
         status_cell = IconWidgetItem(None,icon,val)
-        status_cell.setData(Qt.UserRole, QVariant(val))
+        status_cell.setData(Qt.UserRole, val)
         self.setItem(row, 0, status_cell)
         
         title_cell = ReadOnlyTableWidgetItem(book['title'])
-        title_cell.setData(Qt.UserRole, QVariant(row))
+        title_cell.setData(Qt.UserRole, row)
         self.setItem(row, 1, title_cell)
         
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
@@ -809,7 +908,7 @@ class StoryListTableWidget(QTableWidget):
         books = []
         #print("=========================\nbooks:%s"%self.books)
         for row in range(self.rowCount()):
-            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            rnum = convert_qvariant(self.item(row, 1).data(Qt.UserRole))
             book = self.books[rnum]
             books.append(book)
         return books
@@ -819,9 +918,9 @@ class StoryListTableWidget(QTableWidget):
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        message = '<p>Are you sure you want to remove this book from the list?'
+        message = '<p>'+_('Are you sure you want to remove this book from the list?')
         if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
+            message = '<p>'+_('Are you sure you want to remove the selected %d books from the list?')%len(rows)
         if not confirm(message,'fanfictiondownloader_delete_item', self):
             return
         first_sel_row = self.currentRow()
@@ -847,7 +946,7 @@ class RejectListTableWidget(QTableWidget):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(reject_list))
-        header_labels = ['URL', 'Title', 'Author', 'Note']
+        header_labels = ['URL', _('Title'), _('Author'), _('Note')]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
@@ -875,21 +974,18 @@ class RejectListTableWidget(QTableWidget):
     def populate_table_row(self, row, rej):
 
         url_cell = ReadOnlyTableWidgetItem(rej.url)
-        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
+        url_cell.setData(Qt.UserRole, rej.book_id)
         self.setItem(row, 0, url_cell)
         self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
         self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
         
-        note_cell = EditWithComplete(self)
-        note_cell.lineEdit().mcompleter.model().set_items = \
-            partial(complete_model_set_items_kludge,
-                    note_cell.lineEdit().mcompleter.model())
+        note_cell = EditWithComplete(self,sort_func=lambda x:1)
         
         items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
         note_cell.show_initial_value(rej.note)
         note_cell.set_separator(None)
-        note_cell.setToolTip('Select or Edit Reject Note.')
+        note_cell.setToolTip(_('Select or Edit Reject Note.'))
         self.setCellWidget(row, 3, note_cell)
         
     def remove_selected_rows(self):
@@ -897,9 +993,9 @@ class RejectListTableWidget(QTableWidget):
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        message = '<p>Are you sure you want to remove this URL from the list?'
+        message = '<p>'+_('Are you sure you want to remove this URL from the list?')
         if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+            message = '<p>'+_('Are you sure you want to remove the %d selected URLs from the list?')%len(rows)
         if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
             return
         first_sel_row = self.currentRow()
@@ -917,7 +1013,7 @@ class RejectListTableWidget(QTableWidget):
 class RejectListDialog(SizePersistedDialog):
     def __init__(self, gui, reject_list,
                  rejectreasons=[],
-                 header="List of Books to Reject",
+                 header=_("List of Books to Reject"),
                  icon='rotate-right.png',
                  show_delete=True,
                  show_all_reasons=True,
@@ -930,7 +1026,7 @@ class RejectListDialog(SizePersistedDialog):
         layout = QVBoxLayout(self)
         self.setLayout(layout)
         title_layout = ImageTitleLayout(self, icon, header,
-                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+                                        '<i></i>'+_('FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.'))
         layout.addLayout(title_layout)
         rejects_layout = QHBoxLayout()
         layout.addLayout(rejects_layout)
@@ -944,7 +1040,7 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem)
         
         self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setToolTip(_('Remove selected URL(s) from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
@@ -953,20 +1049,17 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem1)
 
         if show_all_reasons:
-            self.reason_edit = EditWithComplete(self)
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)
             self.reason_edit.show_initial_value('')
             self.reason_edit.set_separator(None)
-            self.reason_edit.setToolTip("This will be added to whatever note you've set for each URL above.")
+            self.reason_edit.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             
             horz = QHBoxLayout()
-            label = QLabel("Add this reason to all URLs added:")
-            label.setToolTip("This will be added to whatever note you've set for each URL above.")
+            label = QLabel(_("Add this reason to all URLs added:"))
+            label.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             horz.addWidget(label)
             horz.addWidget(self.reason_edit)
             horz.insertStretch(-1)
@@ -975,8 +1068,8 @@ class RejectListDialog(SizePersistedDialog):
         options_layout = QHBoxLayout()
 
         if show_delete:
-            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
-            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks = QCheckBox(_('Delete Books (including books without FanFiction URLs)?'),self)
+            self.deletebooks.setToolTip(_("Delete the selected books after adding them to the Rejected URLs list."))
             self.deletebooks.setChecked(True)
             options_layout.addWidget(self.deletebooks)
         
@@ -998,7 +1091,7 @@ class RejectListDialog(SizePersistedDialog):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
             url = unicode(self.rejects_table.item(row, 0).text()).strip()
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id =convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             title = unicode(self.rejects_table.item(row, 1).text()).strip()
             auth = unicode(self.rejects_table.item(row, 2).text()).strip()
             note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
@@ -1008,7 +1101,7 @@ class RejectListDialog(SizePersistedDialog):
     def get_reject_list_ids(self):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id = convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             if book_id:
                 rejectrows.append(book_id)
         return rejectrows
@@ -1023,14 +1116,15 @@ class RejectListDialog(SizePersistedDialog):
     def get_deletebooks(self):
         return self.deletebooks.isChecked()
 
-class EditTextDialog(QDialog):
+class EditTextDialog(SizePersistedDialog):
 
     def __init__(self, parent, text,
                  icon=None, title=None, label=None, tooltip=None,
-                 rejectreasons=[],reasonslabel=None
+                 rejectreasons=[],reasonslabel=None,
+                 save_size_name='ffdl:edit text dialog',
                  ):
-        QDialog.__init__(self, parent)
-        self.resize(600, 500)
+        SizePersistedDialog.__init__(self, parent, save_size_name)
+
         self.l = QVBoxLayout()
         self.setLayout(self.l)
         self.label = QLabel(label)
@@ -1050,11 +1144,7 @@ class EditTextDialog(QDialog):
             self.textedit.setToolTip(tooltip)
 
         if rejectreasons or reasonslabel:
-            self.reason_edit = EditWithComplete(self)
-
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)
@@ -1076,6 +1166,9 @@ class EditTextDialog(QDialog):
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
         self.l.addWidget(button_box)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
 
     def get_plain_text(self):
         return unicode(self.textedit.toPlainText())
@@ -1083,3 +1176,304 @@ class EditTextDialog(QDialog):
     def get_reason_text(self):
         return unicode(self.reason_edit.currentText()).strip()
     
+class IniTextDialog(SizePersistedDialog):
+
+    def __init__(self, parent, text,
+                 icon=None, title=None, label=None,
+                 use_find=False,
+                 read_only=False,
+                 save_size_name='ffdl:ini text dialog',
+                 ):
+        SizePersistedDialog.__init__(self, parent, save_size_name)
+        
+        self.keys=dict()
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel(label)
+        if title:
+            self.setWindowTitle(title)
+        if icon:
+            self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.textedit = QTextEdit(self)
+
+        highlighter = IniHighlighter(self.textedit,
+                                     sections=get_valid_sections(),
+                                     keywords=get_valid_keywords(),
+                                     entries=get_valid_entries(),
+                                     entry_keywords=get_valid_entry_keywords(),
+                                     )
+        
+        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        try:
+            self.textedit.setFont(QFont("Courier",
+                                   parent.font().pointSize()+1))
+        except Exception as e:
+            logger.error("Couldn't get font: %s"%e)
+
+        self.textedit.setReadOnly(read_only)
+
+        self.textedit.setText(text)
+        self.l.addWidget(self.textedit)
+
+        self.lastStart = 0
+
+        if use_find:
+
+            findtooltip=_('Search for string in edit box.')
+            
+            horz = QHBoxLayout()
+            label = QLabel(_('Find:'))
+
+            label.setToolTip(findtooltip)
+                
+            # Button to search the document for something
+            self.findButton = QtGui.QPushButton(_('Find'),self)
+            self.findButton.clicked.connect(self.find)
+            self.findButton.setToolTip(findtooltip)
+        
+            # The field into which to type the query
+            self.findField = QLineEdit(self)
+            self.findField.setToolTip(findtooltip)
+            self.findField.returnPressed.connect(self.findButton.setFocus)
+            
+            # Case Sensitivity option
+            self.caseSens = QtGui.QCheckBox(_('Case sensitive'),self)
+            self.caseSens.setToolTip(_("Search for case sensitive string; don't treat Harry, HARRY and harry all the same."))
+            
+            horz.addWidget(label)
+            horz.addWidget(self.findField)
+            horz.addWidget(self.findButton)
+            horz.addWidget(self.caseSens)
+            
+            self.l.addLayout(horz)
+
+            self.addCtrlKeyPress(QtCore.Qt.Key_F,self.findFocus)
+            self.addCtrlKeyPress(QtCore.Qt.Key_G,self.find)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+
+    def accept(self):
+        from ffdl_util import test_config
+
+        # print("in accept")
+        errors = test_config(self.get_plain_text())
+
+        retry = False
+        if errors:
+            d = ViewLog(self,
+                        _('Go back to fix errors?'),
+                        errors)
+            retry = d.exec_() == d.Accepted
+            
+            # print("retry:%s"%retry)
+        
+        if retry:
+            lineno=d.get_lineno()
+            if lineno:
+                # print("go to lineno (%s) here"%lineno)
+                self.select_line(lineno)
+        else:
+            # print("call parent accept")
+            return SizePersistedDialog.accept(self)
+
+    def addCtrlKeyPress(self,key,func):
+        # print("addKeyPress: key(0x%x)"%key)
+        # print("control: 0x%x"%QtCore.Qt.ControlModifier)
+        self.keys[key]=func
+
+    def keyPressEvent(self, event):
+        # print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers()))
+        if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys:
+            func = self.keys[event.key()]
+            return func()
+        else:
+            return SizePersistedDialog.keyPressEvent(self, event)
+    
+    def get_plain_text(self):
+        return unicode(self.textedit.toPlainText())
+
+    def findFocus(self):
+        # print("findFocus called")
+        self.findField.setFocus()
+        self.findField.selectAll()        
+        
+    def find(self):
+
+        #print("find self.lastStart:%s"%self.lastStart)
+        
+        # Grab the parent's text
+        text = self.textedit.toPlainText()
+
+        # And the text to find
+        query = self.findField.text()
+
+        if not self.caseSens.isChecked():
+            text = text.lower()
+            query = query.lower()
+
+        # Use normal string search to find the query from the
+        # last starting position
+        self.lastStart = text.find(query,self.lastStart + 1)
+        # If the find() method didn't return -1 (not found)
+        
+        if self.lastStart >= 0:
+            end = self.lastStart + len(query)
+            self.moveCursor(self.lastStart,end)
+        else:
+            # Make the next search start from the begining again
+            self.lastStart = 0
+            self.textedit.moveCursor(self.textedit.textCursor().Start)
+                    
+    def moveCursor(self,start,end):
+
+        # We retrieve the QTextCursor object from the parent's QTextEdit
+        cursor = self.textedit.textCursor()
+
+        # Then we set the position to the beginning of the last match
+        cursor.setPosition(start)
+
+        # Next we move the Cursor by over the match and pass the KeepAnchor parameter
+        # which will make the cursor select the the match's text
+        cursor.movePosition(cursor.Right,cursor.KeepAnchor,end - start)
+
+        # And finally we set this new cursor as the parent's
+        self.textedit.setTextCursor(cursor)
+
+    def select_line(self,lineno):
+        
+        # We retrieve the QTextCursor object from the parent's QTextEdit
+        cursor = self.textedit.textCursor()
+
+        # Then we set the position to the beginning of the buffer
+        cursor.setPosition(0)
+
+        # Next we move the Cursor down lineno times
+        cursor.movePosition(cursor.Down,cursor.MoveAnchor,lineno-1)
+
+        # Next we move the Cursor to the end of the line
+        cursor.movePosition(cursor.EndOfLine,cursor.KeepAnchor,1)
+
+        # And finally we set this new cursor as the parent's
+        self.textedit.setTextCursor(cursor)
+
+
+class ViewLog(SizePersistedDialog):
+
+    def label_clicked(self, event, lineno=None):
+        self.lineno = lineno
+        # print("lineno set to: %s"%lineno)
+        self.accept()
+
+    def get_lineno(self):
+        return self.lineno
+        
+    def __init__(self, parent, title, errors, 
+                 save_size_name='ffdl:view log dialog',):
+        SizePersistedDialog.__init__(self, parent,save_size_name)
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+
+        label = QLabel(_('Click an error below to return to Editing directly on that line:'))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+
+        self.lineno = None
+
+        ## error = (lineno, msg)
+        for (lineno, error_msg) in errors:
+            # print('adding label for error:%s: %s'%(lineno, error_msg))
+            label = QLabel('%s: %s'%(lineno, error_msg))
+            label.setWordWrap(True)
+            label.setStyleSheet("QLabel { margin-left: 2em; color : blue; } QLabel:hover { color: red; }");
+            label.setToolTip(_('Click to go to line %s')%lineno)
+            label.mouseReleaseEvent = partial(self.label_clicked, lineno=lineno)
+            self.l.addWidget(label)
+        
+        # html='<p>'+'</p><p>'.join([ '(lineno: %s) %s'%e for e in errors ])+'</p>'
+        
+        # self.tb = QTextBrowser(self)
+        # self.tb.setFont(QFont("Courier",
+        #                       parent.font().pointSize()+1))
+        # self.tb.setHtml(html)
+        # l.addWidget(self.tb)
+
+        self.l.insertStretch(-1)
+
+        horz = QHBoxLayout()
+
+        editagain = QPushButton(_('Return to Editing'), self)
+        editagain.clicked.connect(self.accept)
+        horz.addWidget(editagain)
+
+        saveanyway = QPushButton(_('Save Anyway'), self)
+        saveanyway.clicked.connect(self.reject)
+        horz.addWidget(saveanyway)
+        
+        l.addLayout(horz)
+        self.setModal(False)
+        self.setWindowTitle(title)
+        self.setWindowIcon(QIcon(I('debug.png')))
+        #self.show()
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+
+    def copy_to_clipboard(self):
+        txt = self.tb.toPlainText()
+        QApplication.clipboard().setText(txt)
+
+class EmailPassDialog(QDialog):
+    '''
+    Need to collect Pass for imap.
+    '''
+    def __init__(self, gui, user):
+        QDialog.__init__(self, gui)
+        self.status=False
+
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+
+        self.setWindowTitle(_('Password'))
+        self.l.addWidget(QLabel(_("Enter Email Password for %s:")%user),0,0,1,2)
+   
+        # self.l.addWidget(QLabel(_("Password:")),1,0)
+        self.passwd = QLineEdit(self)
+        self.passwd.setEchoMode(QLineEdit.Password)
+        self.l.addWidget(self.passwd,1,0,1,2)
+
+        self.ok_button = QPushButton(_('OK'), self)
+        self.ok_button.clicked.connect(self.ok)
+        self.l.addWidget(self.ok_button,2,0)
+
+        self.cancel_button = QPushButton(_('Cancel'), self)
+        self.cancel_button.clicked.connect(self.cancel)
+        self.l.addWidget(self.cancel_button,2,1)
+
+        # set stretch factors the same.
+        self.l.setColumnStretch(0,1)
+        self.l.setColumnStretch(1,1)
+
+        self.resize(self.sizeHint())
+
+    def ok(self):
+        self.status=True
+        self.hide()
+
+    def cancel(self):
+        self.status=False
+        self.hide()
+
+    def get_pass(self):
+        return u"%s"%self.passwd.text()
+
+    def get_remember(self):
+        return self.remember_pass.isChecked()

calibre-plugin/ffdl_util.py

@@ -8,10 +8,14 @@ __copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 from StringIO import StringIO
+from ConfigParser import ParsingError
+
+import logging
+logger = logging.getLogger(__name__)
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
-from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs
 
 def get_ffdl_personalini():
     if prefs['includeimages']:
@@ -31,7 +35,7 @@ def get_ffdl_config(url,fileform="epub",personalini=None):
     try:
         site = adapters.getConfigSectionFor(url)
     except Exception as e:
-        print("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
+        logger.debug("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
     configuration = Configuration(site,fileform)
     configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
     configuration.readfp(StringIO(personalini))
@@ -40,4 +44,14 @@ def get_ffdl_config(url,fileform="epub",personalini=None):
 
 def get_ffdl_adapter(url,fileform="epub",personalini=None):
     return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
+
+def test_config(initext):
+    try:
+        configini = get_ffdl_config("test1.com?sid=555",
+                                    personalini=initext)
+        errors = configini.test_config()
+    except ParsingError as pe:
+        errors = pe.errors
     
+    return errors
+

calibre-plugin/inihighlighter.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2014, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import re
+
+try:
+    from PyQt5.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush, QFont)
+except ImportError as e:
+    from PyQt4.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush, QFont)
+
+# r'add_to_+key
+
+
+    
+class IniHighlighter(QSyntaxHighlighter):
+    '''
+    QSyntaxHighlighter class for use with QTextEdit for highlighting
+    ini config files.
+    '''
+    
+    def __init__( self, parent, sections=[], keywords=[], entries=[], entry_keywords=[] ):
+        QSyntaxHighlighter.__init__( self, parent )
+        self.parent = parent
+        
+        self.highlightingRules = []
+
+        if entries:
+            # *known* entries
+            reentries = r'('+(r'|'.join(entries))+r')'
+            self.highlightingRules.append( HighlightingRule( r"\b"+reentries+r"\b", Qt.darkGreen ) )
+
+        # true/false -- just to be nice.
+        self.highlightingRules.append( HighlightingRule( r"\b(true|false)\b", Qt.darkGreen ) )
+        
+        # *all* keywords -- change known later.
+        self.errorRule = HighlightingRule( r"^[^:=\s][^:=]*[:=]", Qt.red )
+        self.highlightingRules.append( self.errorRule )
+
+        # *all* entry keywords -- change known later.
+        reentrykeywords = r'('+(r'|'.join([ e % r'[a-zA-Z0-9_]+' for e in entry_keywords ]))+r')'
+        self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+reentrykeywords+r"\s*[:=]", Qt.darkMagenta ) )
+
+        if entries: # separate from known entries so entry named keyword won't be masked.
+            # *known* entry keywords
+            reentrykeywords = r'('+(r'|'.join([ e % reentries for e in entry_keywords ]))+r')'
+            self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+reentrykeywords+r"\s*[:=]", Qt.blue ) )
+
+        # *known* keywords
+        rekeywords = r'('+(r'|'.join(keywords))+r')'
+        self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+rekeywords+r"\s*[:=]", Qt.blue ) )
+
+        # *all* sections -- change known later.
+        self.highlightingRules.append( HighlightingRule( r"^\[[^\]]+\].*?$", Qt.red, QFont.Bold, blocknum=1 ) )
+
+        if sections:
+            # *known* sections
+            resections = r'('+(r'|'.join(sections))+r')'
+            resections = resections.replace('.','\.') #escape dots.
+            self.highlightingRules.append( HighlightingRule( r"^\["+resections+r"\]\s*$", Qt.darkBlue, QFont.Bold, blocknum=2 ) )
+
+        # test story sections
+        self.teststoryRule = HighlightingRule( r"^\[teststory:([0-9]+|defaults)\]", Qt.darkCyan, blocknum=3 )
+        self.highlightingRules.append( self.teststoryRule )
+
+        # NOT comments -- but can be custom columns, so don't flag.
+        #self.highlightingRules.append( HighlightingRule( r"(?<!^)#[^\n]*" , Qt.red ) )
+        
+        # comments -- comments must start from column 0.
+        self.commentRule = HighlightingRule( r"^#[^\n]*" , Qt.darkYellow )
+        self.highlightingRules.append( self.commentRule )
+
+    def highlightBlock( self, text ):
+
+        is_comment = False
+        blocknum = self.previousBlockState()
+        for rule in self.highlightingRules:
+            for match in rule.pattern.finditer(text):
+                self.setFormat( match.start(), match.end()-match.start(), rule.highlight )
+                if rule == self.commentRule:
+                    is_comment = True
+                if rule.blocknum > 0:
+                    blocknum = rule.blocknum
+
+        if not is_comment:
+            # unknown section, error all:
+            if blocknum == 1 and blocknum == self.previousBlockState():
+                self.setFormat( 0, len(text), self.errorRule.highlight )
+                
+            # teststory section rules:
+            if blocknum == 3:
+                self.setFormat( 0, len(text), self.teststoryRule.highlight )
+            
+        self.setCurrentBlockState( blocknum )
+
+class HighlightingRule():
+    def __init__( self, pattern, color,
+                  weight=QFont.Normal,
+                  style=Qt.SolidPattern,
+                  blocknum=0):
+        if isinstance(pattern,basestring):
+            self.pattern = re.compile(pattern)
+        else:
+            self.pattern=pattern
+        charfmt = QTextCharFormat()
+        brush = QBrush(color, style)
+        charfmt.setForeground(brush)
+        charfmt.setFontWeight(weight)
+        self.highlight = charfmt
+        self.blocknum=blocknum
+    

calibre-plugin/jobs.py

@@ -4,24 +4,19 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2015, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
-import time, os, traceback
-
-from StringIO import StringIO
+import logging
+logger = logging.getLogger(__name__)
 
+import time, traceback
+    
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
 from calibre.constants import numeric_version as calibre_version
 
-from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
-    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
-
-from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config)
 # ------------------------------------------------------------------------------
 #
 #              Functions to perform downloads using worker jobs
@@ -38,13 +33,13 @@ def do_download_worker(book_list, options,
     '''
     server = Server(pool_size=cpus)
 
-    print(options['version'])
+    logger.info(options['version'])
     total = 0
     alreadybad = []
     # Queue all the jobs
-    print("Adding jobs for URLs:")
+    logger.info("Adding jobs for URLs:")
     for book in book_list:
-        print("%s"%book['url'])
+        logger.info("%s"%book['url'])
         if book['good']:
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
@@ -55,10 +50,6 @@ def do_download_worker(book_list, options,
                               done=None,
                               args=args)
             job._book = book
-            # job._book_id = book_id
-            # job._title = title
-            # job._modified_date = modified_date
-            # job._existing_isbn = existing_isbn
             server.add_job(job)
         else:
             # was already bad before the subprocess ever started.
@@ -66,7 +57,7 @@ def do_download_worker(book_list, options,
     
     # This server is an arbitrary_n job, so there is a notifier available.
     # Set the % complete to a small number to avoid the 'unavailable' indicator
-    notification(0.01, 'Downloading FanFiction Stories')
+    notification(0.01, _('Downloading FanFiction Stories'))
 
     # dequeue the job results as they arrive, saving the results
     count = 0
@@ -78,28 +69,20 @@ def do_download_worker(book_list, options,
         if not job.is_finished:
             continue
         # A job really finished. Get the information.
-        output_book = job.result
-        #print("output_book:%s"%output_book)
         book_list.remove(job._book)
         book_list.append(job.result)
         book_id = job._book['calibre_id']
-        #title = job._title
         count = count + 1
         notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
         # Add this job's output to the current log
-        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        print(job.details)
+        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        logger.info(job.details)
 
         if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            print("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    print("%s %s"%(book['title'],book['url']))
-            print("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    print("%s %s"%(book['title'],book['url']))
+            logger.info("\n"+_("Successful:")+"\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\n"+_("Unsuccessful:")+"\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
             break
 
     server.close()
@@ -109,133 +92,161 @@ def do_download_worker(book_list, options,
 
 def do_download_for_worker(book,options,notification=lambda x,y:x):
     '''
-    Child job, to extract isbn from formats for this specific book,
-    when run as a worker job
+    Child job, to download story when run as a worker job
     '''
-    try:
-        book['comment'] = 'Download started...'
 
-        configuration = get_ffdl_config(book['url'],
-                                        options['fileform'],
-                                        options['personal.ini'])
+    from calibre_plugins.fanfictiondownloader_plugin import FanFictionDownLoaderBase
+    ffdlbase = FanFictionDownLoaderBase(options['plugin_path'])
+    with ffdlbase:
         
-        if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-            configuration.set("overrides","never_make_cover","true")
-
-        # images only for epub, even if the user mistakenly turned it
-        # on else where.
-        if options['fileform'] not in ("epub","html"):
-            configuration.set("overrides","include_images","false")
+        from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
+                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+        from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
+        from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
         
-        adapter = adapters.getAdapter(configuration,book['url'])
-        adapter.is_adult = book['is_adult'] 
-        adapter.username = book['username'] 
-        adapter.password = book['password']
-        adapter.setChaptersRange(book['begin'],book['end'])
+        from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config)
         
-        story = adapter.getStoryMetadataOnly()
-        if 'calibre_series' in book:
-            adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
+        try:
+            book['comment'] = _('Download started...')
             
-        # set PI version instead of default.
-        if 'version' in options:
-            story.setMetadata('version',options['version'])
+            configuration = get_ffdl_config(book['url'],
+                                            options['fileform'],
+                                            options['personal.ini'])
+
+            if configuration.getConfig('use_ssl_unverified_context'):
+                ## monkey patch to avoid SSL bug.  dupliated from
+                ## ffdl_plugin.py because bg jobs run in own process
+                ## space.
+                import ssl
+                if hasattr(ssl, '_create_unverified_context'):
+                    ssl._create_default_https_context = ssl._create_unverified_context
+    
+            if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+                configuration.set("overrides","never_make_cover","true")
+    
+            # images only for epub, html, even if the user mistakenly
+            # turned it on else where.
+            if options['fileform'] not in ("epub","html"):
+                configuration.set("overrides","include_images","false")
             
-        writer = writers.getWriter(options['fileform'],configuration,adapter)
-
-        outfile = book['outfile']
-
-        ## No need to download at all.  Shouldn't ever get down here.
-        if options['collision'] in (CALIBREONLY):
-            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
-            book['comment'] = 'Metadata collected.'
+            adapter = adapters.getAdapter(configuration,book['url'])
+            adapter.is_adult = book['is_adult'] 
+            adapter.username = book['username'] 
+            adapter.password = book['password']
+            adapter.setChaptersRange(book['begin'],book['end'])
             
-        ## checks were done earlier, it's new or not dup or newer--just write it.
-        elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
-                ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
-
-            # preserve logfile even on overwrite.
-            if 'epub_for_update' in book:
-                (urlignore,
-                 chaptercountignore,
-                 oldchaptersignore,
-                 oldimgsignore,
-                 oldcoverignore,
-                 calibrebookmarkignore,
-                 # only logfile set in adapter, so others aren't used.
+            adapter.load_cookiejar(options['cookiejarfile'])
+            logger.debug("cookiejar:%s"%adapter.cookiejar)
+            adapter.set_pagecache(options['pagecache'])
+            
+            story = adapter.getStoryMetadataOnly()
+            if 'calibre_series' in book:
+                adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
+                
+            # set PI version instead of default.
+            if 'version' in options:
+                story.setMetadata('version',options['version'])
+                
+            writer = writers.getWriter(options['fileform'],configuration,adapter)
+    
+            outfile = book['outfile']
+    
+            ## No need to download at all.  Shouldn't ever get down here.
+            if options['collision'] in (CALIBREONLY):
+                logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+                book['comment'] = 'Metadata collected.'
+                
+            ## checks were done earlier, it's new or not dup or newer--just write it.
+            elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
+                    ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
+    
+                # preserve logfile even on overwrite.
+                if 'epub_for_update' in book:
+                    (urlignore,
+                     chaptercountignore,
+                     oldchaptersignore,
+                     oldimgsignore,
+                     oldcoverignore,
+                     calibrebookmarkignore,
+                     # only logfile set in adapter, so others aren't used.
+                     adapter.logfile) = get_update_data(book['epub_for_update'])
+    
+                    # change the existing entries id to notid so
+                    # write_epub writes a whole new set to indicate overwrite.
+                    if adapter.logfile:
+                        adapter.logfile = adapter.logfile.replace("span id","span notid")
+                
+                logger.info("write to %s"%outfile)
+                writer.writeStory(outfilename=outfile, forceOverwrite=True)
+                book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+                
+            ## checks were done earlier, just update it.
+            elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+    
+                # update now handled by pre-populating the old images and
+                # chapters in the adapter rather than merging epubs.
+                urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
+                (url,
+                 chaptercount,
+                 adapter.oldchapters,
+                 adapter.oldimgs,
+                 adapter.oldcover,
+                 adapter.calibrebookmark,
                  adapter.logfile) = get_update_data(book['epub_for_update'])
-
-                # change the existing entries id to notid so
-                # write_epub writes a whole new set to indicate overwrite.
-                if adapter.logfile:
-                    adapter.logfile = adapter.logfile.replace("span id","span notid")
+    
+                # dup handling from ffdl_plugin needed for anthology updates.
+                if options['collision'] == UPDATE:
+                    if chaptercount == urlchaptercount:
+                        book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
+                        book['outfile'] = book['epub_for_update'] # for anthology merge ops.
+                        return book
+    
+                # dup handling from ffdl_plugin needed for anthology updates.
+                if chaptercount > urlchaptercount:
+                    raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
+    
+                if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
+                        and adapter.getConfig("do_update_hook"):
+                    chaptercount = adapter.hookForUpdates(chaptercount)
+    
+                logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+                logger.info("write to %s"%outfile)
+    
+                writer.writeStory(outfilename=outfile, forceOverwrite=True)
+                
+                book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
+                    (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
             
-            print("write to %s"%outfile)
-            writer.writeStory(outfilename=outfile, forceOverwrite=True)
-            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+            if options['smarten_punctuation'] and options['fileform'] == "epub" \
+                    and calibre_version >= (0, 9, 39):
+                # for smarten punc
+                from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                from calibre.utils.logging import Log
+                from collections import namedtuple
+
+                # do smarten_punctuation from calibre's polish feature
+                data = {'smarten_punctuation':True}
+                opts = ALL_OPTS.copy()
+                opts.update(data)
+                O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                opts = O(**opts)
+                
+                log = Log(level=Log.DEBUG)
+                # report = []
+                polish({outfile:outfile}, opts, log, logger.info) # report.append
             
-        ## checks were done earlier, just update it.
-        elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-
-            # update now handled by pre-populating the old images and
-            # chapters in the adapter rather than merging epubs.
-            urlchaptercount = int(story.getMetadata('numChapters'))
-            (url,
-             chaptercount,
-             adapter.oldchapters,
-             adapter.oldimgs,
-             adapter.oldcover,
-             adapter.calibrebookmark,
-             adapter.logfile) = get_update_data(book['epub_for_update'])
-
-            # dup handling from ffdl_plugin needed for anthology updates.
-            if options['collision'] == UPDATE:
-                if chaptercount == urlchaptercount:
-                    book['comment']="Already contains %d chapters.  Reuse as is."%chaptercount
-                    book['outfile'] = book['epub_for_update'] # for anthology merge ops.
-                    return book
-
-            # dup handling from ffdl_plugin needed for anthology updates.
-            if chaptercount > urlchaptercount:
-                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
-                            
-            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            print("write to %s"%outfile)
-
-            writer.writeStory(outfilename=outfile, forceOverwrite=True)
+        except NotGoingToDownload as d:
+            book['good']=False
+            book['comment']=unicode(d)
+            book['icon'] = d.icon
+    
+        except Exception as e:
+            book['good']=False
+            book['comment']=unicode(e)
+            book['icon']='dialog_error.png'
+            book['status'] = 'Error'
+            logger.info("Exception: %s:%s"%(book,unicode(e)))
+            traceback.print_exc()
             
-            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
-                (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
-        
-        if options['smarten_punctuation'] and options['fileform'] == "epub" \
-                and calibre_version >= (0, 9, 39):
-            # do smarten_punctuation from calibre's polish feature
-            from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
-            from calibre.utils.logging import Log
-            from collections import namedtuple
-        
-            data = {'smarten_punctuation':True}
-            opts = ALL_OPTS.copy()
-            opts.update(data)
-            O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
-            opts = O(**opts)
-            
-            log = Log(level=Log.DEBUG)
-            # report = []
-            polish({outfile:outfile}, opts, log, print) # report.append
-        
-    except NotGoingToDownload as d:
-        book['good']=False
-        book['comment']=unicode(d)
-        book['icon'] = d.icon
-
-    except Exception as e:
-        book['good']=False
-        book['comment']=unicode(e)
-        book['icon']='dialog_error.png'
-        book['status'] = 'Error'
-        print("Exception: %s:%s"%(book,unicode(e)))
-        traceback.print_exc()
-        
-    #time.sleep(10)
+        #time.sleep(10)
     return book

calibre-plugin/prefs.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2013, Jim Miller'
+__copyright__ = '2015, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import copy
@@ -25,6 +25,7 @@ default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
 Dup from another site'''
+default_prefs['reject_always'] = False
 
 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
@@ -32,7 +33,9 @@ default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
 default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
+default_prefs['mark'] = False
 default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
@@ -45,6 +48,7 @@ default_prefs['checkforseriesurlid'] = True
 default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
 default_prefs['smarten_punctuation'] = False
+default_prefs['show_est_time'] = False
 
 default_prefs['send_lists'] = ''
 default_prefs['read_lists'] = ''
@@ -55,8 +59,10 @@ default_prefs['addtolistsonread'] = False
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False
 
 default_prefs['countpagesstats'] = []
+default_prefs['wordcountmissing'] = False
 
 default_prefs['errorcol'] = ''
 default_prefs['custom_cols'] = {}
@@ -65,6 +71,13 @@ default_prefs['allow_custcol_from_ini'] = True
 
 default_prefs['std_cols_newonly'] = {}
 
+default_prefs['imapserver'] = ''
+default_prefs['imapuser'] = ''
+default_prefs['imappass'] = ''
+default_prefs['imapsessionpass'] = False
+default_prefs['imapfolder'] = 'INBOX'
+default_prefs['imapmarkread'] = True
+
 # This is where all preferences for this plugin *were* stored
 # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
 # in a global namespace, so make it as unique as possible.

chardet/__init__.py

@@ -3,22 +3,28 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-__version__ = "2.0.1"
+__version__ = "2.3.0"
+from sys import version_info
+
 
 def detect(aBuf):
-    import universaldetector
+    if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or
+            (version_info >= (3, 0) and not isinstance(aBuf, bytes))):
+        raise ValueError('Expected a bytes object, not a unicode object')
+
+    from . import universaldetector
     u = universaldetector.UniversalDetector()
     u.reset()
     u.feed(aBuf)

chardet/big5freq.py

@@ -1,11 +1,11 @@
 ######################## BEGIN LICENSE BLOCK ########################
 # The Original Code is Mozilla Communicator client code.
-# 
+#
 # The Initial Developer of the Original Code is
 # Netscape Communications Corporation.
 # Portions created by the Initial Developer are Copyright (C) 1998
 # the Initial Developer. All Rights Reserved.
-# 
+#
 # Contributor(s):
 #   Mark Pilgrim - port to Python
 #
@@ -13,12 +13,12 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
@@ -26,18 +26,18 @@
 ######################### END LICENSE BLOCK #########################
 
 # Big5 frequency table
-# by Taiwan's Mandarin Promotion Council 
+# by Taiwan's Mandarin Promotion Council
 # <http://www.edu.tw:81/mandr/>
-# 
+#
 # 128  --> 0.42261
 # 256  --> 0.57851
 # 512  --> 0.74851
 # 1024 --> 0.89384
 # 2048 --> 0.97583
-# 
+#
 # Ideal Distribution Ratio = 0.74851/(1-0.74851) =2.98
 # Random Distribution Ration = 512/(5401-512)=0.105
-# 
+#
 # Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR
 
 BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75
@@ -45,7 +45,7 @@ BIG5_TYPICAL_DISTRIBUTION_RATIO = 0.75
 #Char to FreqOrder table
 BIG5_TABLE_SIZE = 5376
 
-Big5CharToFreqOrder = ( \
+Big5CharToFreqOrder = (
    1,1801,1506, 255,1431, 198,   9,  82,   6,5008, 177, 202,3681,1256,2821, 110, #   16
 3814,  33,3274, 261,  76,  44,2114,  16,2946,2187,1176, 659,3971,  26,3451,2653, #   32
 1198,3972,3350,4202, 410,2215, 302, 590, 361,1964,   8, 204,  58,4510,5009,1932, #   48
@@ -921,3 +921,5 @@ Big5CharToFreqOrder = ( \
 13936,13937,13938,13939,13940,13941,13942,13943,13944,13945,13946,13947,13948,13949,13950,13951, #13952
 13952,13953,13954,13955,13956,13957,13958,13959,13960,13961,13962,13963,13964,13965,13966,13967, #13968
 13968,13969,13970,13971,13972) #13973
+
+# flake8: noqa

chardet/big5prober.py

@@ -1,11 +1,11 @@
 ######################## BEGIN LICENSE BLOCK ########################
 # The Original Code is Mozilla Communicator client code.
-# 
+#
 # The Initial Developer of the Original Code is
 # Netscape Communications Corporation.
 # Portions created by the Initial Developer are Copyright (C) 1998
 # the Initial Developer. All Rights Reserved.
-# 
+#
 # Contributor(s):
 #   Mark Pilgrim - port to Python
 #
@@ -13,22 +13,23 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-from mbcharsetprober import MultiByteCharSetProber
-from codingstatemachine import CodingStateMachine
-from chardistribution import Big5DistributionAnalysis
-from mbcssm import Big5SMModel
+from .mbcharsetprober import MultiByteCharSetProber
+from .codingstatemachine import CodingStateMachine
+from .chardistribution import Big5DistributionAnalysis
+from .mbcssm import Big5SMModel
+
 
 class Big5Prober(MultiByteCharSetProber):
     def __init__(self):

chardet/chardetect.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+"""
+Script which takes one or more file paths and reports on their detected
+encodings
+
+Example::
+
+    % chardetect somefile someotherfile
+    somefile: windows-1252 with confidence 0.5
+    someotherfile: ascii with confidence 1.0
+
+If no paths are provided, it takes its input from stdin.
+
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import sys
+from io import open
+
+from chardet import __version__
+from chardet.universaldetector import UniversalDetector
+
+
+def description_of(lines, name='stdin'):
+    """
+    Return a string describing the probable encoding of a file or
+    list of strings.
+
+    :param lines: The lines to get the encoding of.
+    :type lines: Iterable of bytes
+    :param name: Name of file or collection of lines
+    :type name: str
+    """
+    u = UniversalDetector()
+    for line in lines:
+        u.feed(line)
+    u.close()
+    result = u.result
+    if result['encoding']:
+        return '{0}: {1} with confidence {2}'.format(name, result['encoding'],
+                                                     result['confidence'])
+    else:
+        return '{0}: no result'.format(name)
+
+
+def main(argv=None):
+    '''
+    Handles command line arguments and gets things started.
+
+    :param argv: List of arguments, as if specified on the command-line.
+                 If None, ``sys.argv[1:]`` is used instead.
+    :type argv: list of str
+    '''
+    # Get command line arguments
+    parser = argparse.ArgumentParser(
+        description="Takes one or more file paths and reports their detected \
+                     encodings",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        conflict_handler='resolve')
+    parser.add_argument('input',
+                        help='File whose encoding we would like to determine.',
+                        type=argparse.FileType('rb'), nargs='*',
+                        default=[sys.stdin])
+    parser.add_argument('--version', action='version',
+                        version='%(prog)s {0}'.format(__version__))
+    args = parser.parse_args(argv)
+
+    for f in args.input:
+        if f.isatty():
+            print("You are running chardetect interactively. Press " +
+                  "CTRL-D twice at the start of a blank line to signal the " +
+                  "end of your input. If you want help, run chardetect " +
+                  "--help\n", file=sys.stderr)
+        print(description_of(f, f.name))
+
+
+if __name__ == '__main__':
+    main()

chardet/chardistribution.py

@@ -1,11 +1,11 @@
 ######################## BEGIN LICENSE BLOCK ########################
 # The Original Code is Mozilla Communicator client code.
-# 
+#
 # The Initial Developer of the Original Code is
 # Netscape Communications Corporation.
 # Portions created by the Initial Developer are Copyright (C) 1998
 # the Initial Developer. All Rights Reserved.
-# 
+#
 # Contributor(s):
 #   Mark Pilgrim - port to Python
 #
@@ -13,47 +13,63 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants
-from euctwfreq import EUCTWCharToFreqOrder, EUCTW_TABLE_SIZE, EUCTW_TYPICAL_DISTRIBUTION_RATIO
-from euckrfreq import EUCKRCharToFreqOrder, EUCKR_TABLE_SIZE, EUCKR_TYPICAL_DISTRIBUTION_RATIO
-from gb2312freq import GB2312CharToFreqOrder, GB2312_TABLE_SIZE, GB2312_TYPICAL_DISTRIBUTION_RATIO
-from big5freq import Big5CharToFreqOrder, BIG5_TABLE_SIZE, BIG5_TYPICAL_DISTRIBUTION_RATIO
-from jisfreq import JISCharToFreqOrder, JIS_TABLE_SIZE, JIS_TYPICAL_DISTRIBUTION_RATIO
+from .euctwfreq import (EUCTWCharToFreqOrder, EUCTW_TABLE_SIZE,
+                        EUCTW_TYPICAL_DISTRIBUTION_RATIO)
+from .euckrfreq import (EUCKRCharToFreqOrder, EUCKR_TABLE_SIZE,
+                        EUCKR_TYPICAL_DISTRIBUTION_RATIO)
+from .gb2312freq import (GB2312CharToFreqOrder, GB2312_TABLE_SIZE,
+                         GB2312_TYPICAL_DISTRIBUTION_RATIO)
+from .big5freq import (Big5CharToFreqOrder, BIG5_TABLE_SIZE,
+                       BIG5_TYPICAL_DISTRIBUTION_RATIO)
+from .jisfreq import (JISCharToFreqOrder, JIS_TABLE_SIZE,
+                      JIS_TYPICAL_DISTRIBUTION_RATIO)
+from .compat import wrap_ord
 
 ENOUGH_DATA_THRESHOLD = 1024
 SURE_YES = 0.99
 SURE_NO = 0.01
+MINIMUM_DATA_THRESHOLD = 3
+
 
 class CharDistributionAnalysis:
     def __init__(self):
-        self._mCharToFreqOrder = None # Mapping table to get frequency order from char order (get from GetOrder())
-        self._mTableSize = None # Size of above table
-        self._mTypicalDistributionRatio = None # This is a constant value which varies from language to language, used in calculating confidence.  See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail.
+        # Mapping table to get frequency order from char order (get from
+        # GetOrder())
+        self._mCharToFreqOrder = None
+        self._mTableSize = None  # Size of above table
+        # This is a constant value which varies from language to language,
+        # used in calculating confidence.  See
+        # http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html
+        # for further detail.
+        self._mTypicalDistributionRatio = None
         self.reset()
-        
+
     def reset(self):
         """reset analyser, clear any state"""
-        self._mDone = constants.False # If this flag is set to constants.True, detection is done and conclusion has been made
-        self._mTotalChars = 0 # Total characters encountered
-        self._mFreqChars = 0 # The number of characters whose frequency order is less than 512
+        # If this flag is set to True, detection is done and conclusion has
+        # been made
+        self._mDone = False
+        self._mTotalChars = 0  # Total characters encountered
+        # The number of characters whose frequency order is less than 512
+        self._mFreqChars = 0
 
-    def feed(self, aStr, aCharLen):
+    def feed(self, aBuf, aCharLen):
         """feed a character with known length"""
         if aCharLen == 2:
             # we only care about 2-bytes character in our distribution analysis
-            order = self.get_order(aStr)
+            order = self.get_order(aBuf)
         else:
             order = -1
         if order >= 0:
@@ -65,12 +81,14 @@ class CharDistributionAnalysis:
 
     def get_confidence(self):
         """return confidence based on existing data"""
-        # if we didn't receive any character in our consideration range, return negative answer
-        if self._mTotalChars <= 0:
+        # if we didn't receive any character in our consideration range,
+        # return negative answer
+        if self._mTotalChars <= 0 or self._mFreqChars <= MINIMUM_DATA_THRESHOLD:
             return SURE_NO
 
         if self._mTotalChars != self._mFreqChars:
-            r = self._mFreqChars / ((self._mTotalChars - self._mFreqChars) * self._mTypicalDistributionRatio)
+            r = (self._mFreqChars / ((self._mTotalChars - self._mFreqChars)
+                 * self._mTypicalDistributionRatio))
             if r < SURE_YES:
                 return r
 
@@ -78,16 +96,18 @@ class CharDistributionAnalysis:
         return SURE_YES
 
     def got_enough_data(self):
-        # It is not necessary to receive all data to draw conclusion. For charset detection,
-        # certain amount of data is enough
+        # It is not necessary to receive all data to draw conclusion.
+        # For charset detection, certain amount of data is enough
         return self._mTotalChars > ENOUGH_DATA_THRESHOLD
 
-    def get_order(self, aStr):
-        # We do not handle characters based on the original encoding string, but 
-        # convert this encoding string to a number, here called order.
-        # This allows multiple encodings of a language to share one frequency table.
+    def get_order(self, aBuf):
+        # We do not handle characters based on the original encoding string,
+        # but convert this encoding string to a number, here called order.
+        # This allows multiple encodings of a language to share one frequency
+        # table.
         return -1
-    
+
+
 class EUCTWDistributionAnalysis(CharDistributionAnalysis):
     def __init__(self):
         CharDistributionAnalysis.__init__(self)
@@ -95,16 +115,18 @@ class EUCTWDistributionAnalysis(CharDistributionAnalysis):
         self._mTableSize = EUCTW_TABLE_SIZE
         self._mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
 
-    def get_order(self, aStr):
-        # for euc-TW encoding, we are interested 
+    def get_order(self, aBuf):
+        # for euc-TW encoding, we are interested
         #   first  byte range: 0xc4 -- 0xfe
         #   second byte range: 0xa1 -- 0xfe
         # no validation needed here. State machine has done that
-        if aStr[0] >= '\xC4':
-            return 94 * (ord(aStr[0]) - 0xC4) + ord(aStr[1]) - 0xA1
+        first_char = wrap_ord(aBuf[0])
+        if first_char >= 0xC4:
+            return 94 * (first_char - 0xC4) + wrap_ord(aBuf[1]) - 0xA1
         else:
             return -1
 
+
 class EUCKRDistributionAnalysis(CharDistributionAnalysis):
     def __init__(self):
         CharDistributionAnalysis.__init__(self)
@@ -112,15 +134,17 @@ class EUCKRDistributionAnalysis(CharDistributionAnalysis):
         self._mTableSize = EUCKR_TABLE_SIZE
         self._mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
 
-    def get_order(self, aStr):
-        # for euc-KR encoding, we are interested 
+    def get_order(self, aBuf):
+        # for euc-KR encoding, we are interested
         #   first  byte range: 0xb0 -- 0xfe
         #   second byte range: 0xa1 -- 0xfe
         # no validation needed here. State machine has done that
-        if aStr[0] >= '\xB0':
-            return 94 * (ord(aStr[0]) - 0xB0) + ord(aStr[1]) - 0xA1
+        first_char = wrap_ord(aBuf[0])
+        if first_char >= 0xB0:
+            return 94 * (first_char - 0xB0) + wrap_ord(aBuf[1]) - 0xA1
         else:
-            return -1;
+            return -1
+
 
 class GB2312DistributionAnalysis(CharDistributionAnalysis):
     def __init__(self):
@@ -129,15 +153,17 @@ class GB2312DistributionAnalysis(CharDistributionAnalysis):
         self._mTableSize = GB2312_TABLE_SIZE
         self._mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO
 
-    def get_order(self, aStr):
-        # for GB2312 encoding, we are interested 
+    def get_order(self, aBuf):
+        # for GB2312 encoding, we are interested
         #  first  byte range: 0xb0 -- 0xfe
         #  second byte range: 0xa1 -- 0xfe
         # no validation needed here. State machine has done that
-        if (aStr[0] >= '\xB0') and (aStr[1] >= '\xA1'):
-            return 94 * (ord(aStr[0]) - 0xB0) + ord(aStr[1]) - 0xA1
+        first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
+        if (first_char >= 0xB0) and (second_char >= 0xA1):
+            return 94 * (first_char - 0xB0) + second_char - 0xA1
         else:
-            return -1;
+            return -1
+
 
 class Big5DistributionAnalysis(CharDistributionAnalysis):
     def __init__(self):
@@ -146,19 +172,21 @@ class Big5DistributionAnalysis(CharDistributionAnalysis):
         self._mTableSize = BIG5_TABLE_SIZE
         self._mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO
 
-    def get_order(self, aStr):
-        # for big5 encoding, we are interested 
+    def get_order(self, aBuf):
+        # for big5 encoding, we are interested
         #   first  byte range: 0xa4 -- 0xfe
         #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
         # no validation needed here. State machine has done that
-        if aStr[0] >= '\xA4':
-            if aStr[1] >= '\xA1':
-                return 157 * (ord(aStr[0]) - 0xA4) + ord(aStr[1]) - 0xA1 + 63
+        first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
+        if first_char >= 0xA4:
+            if second_char >= 0xA1:
+                return 157 * (first_char - 0xA4) + second_char - 0xA1 + 63
             else:
-                return 157 * (ord(aStr[0]) - 0xA4) + ord(aStr[1]) - 0x40
+                return 157 * (first_char - 0xA4) + second_char - 0x40
         else:
             return -1
 
+
 class SJISDistributionAnalysis(CharDistributionAnalysis):
     def __init__(self):
         CharDistributionAnalysis.__init__(self)
@@ -166,22 +194,24 @@ class SJISDistributionAnalysis(CharDistributionAnalysis):
         self._mTableSize = JIS_TABLE_SIZE
         self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
 
-    def get_order(self, aStr):
-        # for sjis encoding, we are interested 
+    def get_order(self, aBuf):
+        # for sjis encoding, we are interested
         #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
         #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
         # no validation needed here. State machine has done that
-        if (aStr[0] >= '\x81') and (aStr[0] <= '\x9F'):
-            order = 188 * (ord(aStr[0]) - 0x81)
-        elif (aStr[0] >= '\xE0') and (aStr[0] <= '\xEF'):
-            order = 188 * (ord(aStr[0]) - 0xE0 + 31)
+        first_char, second_char = wrap_ord(aBuf[0]), wrap_ord(aBuf[1])
+        if (first_char >= 0x81) and (first_char <= 0x9F):
+            order = 188 * (first_char - 0x81)
+        elif (first_char >= 0xE0) and (first_char <= 0xEF):
+            order = 188 * (first_char - 0xE0 + 31)
         else:
-            return -1;
-        order = order + ord(aStr[1]) - 0x40
-        if aStr[1] > '\x7F':
-            order =- 1
+            return -1
+        order = order + second_char - 0x40
+        if second_char > 0x7F:
+            order = -1
         return order
 
+
 class EUCJPDistributionAnalysis(CharDistributionAnalysis):
     def __init__(self):
         CharDistributionAnalysis.__init__(self)
@@ -189,12 +219,13 @@ class EUCJPDistributionAnalysis(CharDistributionAnalysis):
         self._mTableSize = JIS_TABLE_SIZE
         self._mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
 
-    def get_order(self, aStr):
-        # for euc-JP encoding, we are interested 
+    def get_order(self, aBuf):
+        # for euc-JP encoding, we are interested
         #   first  byte range: 0xa0 -- 0xfe
         #   second byte range: 0xa1 -- 0xfe
         # no validation needed here. State machine has done that
-        if aStr[0] >= '\xA0':
-            return 94 * (ord(aStr[0]) - 0xA1) + ord(aStr[1]) - 0xa1
+        char = wrap_ord(aBuf[0])
+        if char >= 0xA0:
+            return 94 * (char - 0xA1) + wrap_ord(aBuf[1]) - 0xa1
         else:
             return -1

chardet/charsetgroupprober.py

@@ -25,8 +25,10 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, sys
-from charsetprober import CharSetProber
+from . import constants
+import sys
+from .charsetprober import CharSetProber
+
 
 class CharSetGroupProber(CharSetProber):
     def __init__(self):
@@ -34,35 +36,39 @@ class CharSetGroupProber(CharSetProber):
         self._mActiveNum = 0
         self._mProbers = []
         self._mBestGuessProber = None
-        
+
     def reset(self):
         CharSetProber.reset(self)
         self._mActiveNum = 0
         for prober in self._mProbers:
             if prober:
                 prober.reset()
-                prober.active = constants.True
+                prober.active = True
                 self._mActiveNum += 1
         self._mBestGuessProber = None
 
     def get_charset_name(self):
         if not self._mBestGuessProber:
             self.get_confidence()
-            if not self._mBestGuessProber: return None
+            if not self._mBestGuessProber:
+                return None
 #                self._mBestGuessProber = self._mProbers[0]
         return self._mBestGuessProber.get_charset_name()
 
     def feed(self, aBuf):
         for prober in self._mProbers:
-            if not prober: continue
-            if not prober.active: continue
+            if not prober:
+                continue
+            if not prober.active:
+                continue
             st = prober.feed(aBuf)
-            if not st: continue
+            if not st:
+                continue
             if st == constants.eFoundIt:
                 self._mBestGuessProber = prober
                 return self.get_state()
             elif st == constants.eNotMe:
-                prober.active = constants.False
+                prober.active = False
                 self._mActiveNum -= 1
                 if self._mActiveNum <= 0:
                     self._mState = constants.eNotMe
@@ -78,18 +84,22 @@ class CharSetGroupProber(CharSetProber):
         bestConf = 0.0
         self._mBestGuessProber = None
         for prober in self._mProbers:
-            if not prober: continue
+            if not prober:
+                continue
             if not prober.active:
                 if constants._debug:
-                    sys.stderr.write(prober.get_charset_name() + ' not active\n')
+                    sys.stderr.write(prober.get_charset_name()
+                                     + ' not active\n')
                 continue
             cf = prober.get_confidence()
             if constants._debug:
-                sys.stderr.write('%s confidence = %s\n' % (prober.get_charset_name(), cf))
+                sys.stderr.write('%s confidence = %s\n' %
+                                 (prober.get_charset_name(), cf))
             if bestConf < cf:
                 bestConf = cf
                 self._mBestGuessProber = prober
-        if not self._mBestGuessProber: return 0.0
+        if not self._mBestGuessProber:
+            return 0.0
         return bestConf
 #        else:
 #            self._mBestGuessProber = self._mProbers[0]

chardet/charsetprober.py

@@ -1,11 +1,11 @@
 ######################## BEGIN LICENSE BLOCK ########################
 # The Original Code is Mozilla Universal charset detector code.
-# 
+#
 # The Initial Developer of the Original Code is
 # Netscape Communications Corporation.
 # Portions created by the Initial Developer are Copyright (C) 2001
 # the Initial Developer. All Rights Reserved.
-# 
+#
 # Contributor(s):
 #   Mark Pilgrim - port to Python
 #   Shy Shalom - original C code
@@ -14,27 +14,29 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, re
+from . import constants
+import re
+
 
 class CharSetProber:
     def __init__(self):
         pass
-        
+
     def reset(self):
         self._mState = constants.eDetecting
-    
+
     def get_charset_name(self):
         return None
 
@@ -48,13 +50,13 @@ class CharSetProber:
         return 0.0
 
     def filter_high_bit_only(self, aBuf):
-        aBuf = re.sub(r'([\x00-\x7F])+', ' ', aBuf)
+        aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf)
         return aBuf
-    
+
     def filter_without_english_letters(self, aBuf):
-        aBuf = re.sub(r'([A-Za-z])+', ' ', aBuf)
+        aBuf = re.sub(b'([A-Za-z])+', b' ', aBuf)
         return aBuf
-        
+
     def filter_with_english_letters(self, aBuf):
         # TODO
         return aBuf

chardet/codingstatemachine.py

@@ -13,19 +13,21 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-from constants import eStart, eError, eItsMe
+from .constants import eStart
+from .compat import wrap_ord
+
 
 class CodingStateMachine:
     def __init__(self, sm):
@@ -40,12 +42,15 @@ class CodingStateMachine:
     def next_state(self, c):
         # for each byte we get its class
         # if it is first byte, we also get byte length
-        byteCls = self._mModel['classTable'][ord(c)]
+        # PY3K: aBuf is a byte stream, so c is an int, not a byte
+        byteCls = self._mModel['classTable'][wrap_ord(c)]
         if self._mCurrentState == eStart:
             self._mCurrentBytePos = 0
             self._mCurrentCharLen = self._mModel['charLenTable'][byteCls]
         # from byte's class and stateTable, we get its next state
-        self._mCurrentState = self._mModel['stateTable'][self._mCurrentState * self._mModel['classFactor'] + byteCls]
+        curr_state = (self._mCurrentState * self._mModel['classFactor']
+                      + byteCls)
+        self._mCurrentState = self._mModel['stateTable'][curr_state]
         self._mCurrentBytePos += 1
         return self._mCurrentState
 

chardet/compat.py

@@ -0,0 +1,34 @@
+######################## BEGIN LICENSE BLOCK ########################
+# Contributor(s):
+#   Ian Cordasco - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+import sys
+
+
+if sys.version_info < (3, 0):
+    base_str = (str, unicode)
+else:
+    base_str = (bytes, str)
+
+
+def wrap_ord(a):
+    if sys.version_info < (3, 0) and isinstance(a, base_str):
+        return ord(a)
+    else:
+        return a

chardet/constants.py

@@ -37,11 +37,3 @@ eError = 1
 eItsMe = 2
 
 SHORTCUT_THRESHOLD = 0.95
-
-import __builtin__
-if not hasattr(__builtin__, 'False'):
-    False = 0
-    True = 1
-else:
-    False = __builtin__.False
-    True = __builtin__.True

chardet/cp949prober.py

@@ -0,0 +1,44 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .mbcharsetprober import MultiByteCharSetProber
+from .codingstatemachine import CodingStateMachine
+from .chardistribution import EUCKRDistributionAnalysis
+from .mbcssm import CP949SMModel
+
+
+class CP949Prober(MultiByteCharSetProber):
+    def __init__(self):
+        MultiByteCharSetProber.__init__(self)
+        self._mCodingSM = CodingStateMachine(CP949SMModel)
+        # NOTE: CP949 is a superset of EUC-KR, so the distribution should be
+        #       not different.
+        self._mDistributionAnalyzer = EUCKRDistributionAnalysis()
+        self.reset()
+
+    def get_charset_name(self):
+        return "CP949"

chardet/escprober.py

@@ -13,39 +13,43 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, sys
-from escsm import HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, ISO2022KRSMModel
-from charsetprober import CharSetProber
-from codingstatemachine import CodingStateMachine
+from . import constants
+from .escsm import (HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel,
+                    ISO2022KRSMModel)
+from .charsetprober import CharSetProber
+from .codingstatemachine import CodingStateMachine
+from .compat import wrap_ord
+
 
 class EscCharSetProber(CharSetProber):
     def __init__(self):
         CharSetProber.__init__(self)
-        self._mCodingSM = [ \
+        self._mCodingSM = [
             CodingStateMachine(HZSMModel),
             CodingStateMachine(ISO2022CNSMModel),
             CodingStateMachine(ISO2022JPSMModel),
             CodingStateMachine(ISO2022KRSMModel)
-            ]
+        ]
         self.reset()
 
     def reset(self):
         CharSetProber.reset(self)
         for codingSM in self._mCodingSM:
-            if not codingSM: continue
-            codingSM.active = constants.True
+            if not codingSM:
+                continue
+            codingSM.active = True
             codingSM.reset()
         self._mActiveSM = len(self._mCodingSM)
         self._mDetectedCharset = None
@@ -61,19 +65,22 @@ class EscCharSetProber(CharSetProber):
 
     def feed(self, aBuf):
         for c in aBuf:
+            # PY3K: aBuf is a byte array, so c is an int, not a byte
             for codingSM in self._mCodingSM:
-                if not codingSM: continue
-                if not codingSM.active: continue
-                codingState = codingSM.next_state(c)
+                if not codingSM:
+                    continue
+                if not codingSM.active:
+                    continue
+                codingState = codingSM.next_state(wrap_ord(c))
                 if codingState == constants.eError:
-                    codingSM.active = constants.False
+                    codingSM.active = False
                     self._mActiveSM -= 1
                     if self._mActiveSM <= 0:
                         self._mState = constants.eNotMe
                         return self.get_state()
                 elif codingState == constants.eItsMe:
                     self._mState = constants.eFoundIt
-                    self._mDetectedCharset = codingSM.get_coding_state_machine()
+                    self._mDetectedCharset = codingSM.get_coding_state_machine()  # nopep8
                     return self.get_state()
-                
+
         return self.get_state()

chardet/escsm.py

@@ -0,0 +1,242 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .constants import eStart, eError, eItsMe
+
+HZ_cls = (
+1,0,0,0,0,0,0,0,  # 00 - 07
+0,0,0,0,0,0,0,0,  # 08 - 0f
+0,0,0,0,0,0,0,0,  # 10 - 17
+0,0,0,1,0,0,0,0,  # 18 - 1f
+0,0,0,0,0,0,0,0,  # 20 - 27
+0,0,0,0,0,0,0,0,  # 28 - 2f
+0,0,0,0,0,0,0,0,  # 30 - 37
+0,0,0,0,0,0,0,0,  # 38 - 3f
+0,0,0,0,0,0,0,0,  # 40 - 47
+0,0,0,0,0,0,0,0,  # 48 - 4f
+0,0,0,0,0,0,0,0,  # 50 - 57
+0,0,0,0,0,0,0,0,  # 58 - 5f
+0,0,0,0,0,0,0,0,  # 60 - 67
+0,0,0,0,0,0,0,0,  # 68 - 6f
+0,0,0,0,0,0,0,0,  # 70 - 77
+0,0,0,4,0,5,2,0,  # 78 - 7f
+1,1,1,1,1,1,1,1,  # 80 - 87
+1,1,1,1,1,1,1,1,  # 88 - 8f
+1,1,1,1,1,1,1,1,  # 90 - 97
+1,1,1,1,1,1,1,1,  # 98 - 9f
+1,1,1,1,1,1,1,1,  # a0 - a7
+1,1,1,1,1,1,1,1,  # a8 - af
+1,1,1,1,1,1,1,1,  # b0 - b7
+1,1,1,1,1,1,1,1,  # b8 - bf
+1,1,1,1,1,1,1,1,  # c0 - c7
+1,1,1,1,1,1,1,1,  # c8 - cf
+1,1,1,1,1,1,1,1,  # d0 - d7
+1,1,1,1,1,1,1,1,  # d8 - df
+1,1,1,1,1,1,1,1,  # e0 - e7
+1,1,1,1,1,1,1,1,  # e8 - ef
+1,1,1,1,1,1,1,1,  # f0 - f7
+1,1,1,1,1,1,1,1,  # f8 - ff
+)
+
+HZ_st = (
+eStart,eError,     3,eStart,eStart,eStart,eError,eError,# 00-07
+eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f
+eItsMe,eItsMe,eError,eError,eStart,eStart,     4,eError,# 10-17
+     5,eError,     6,eError,     5,     5,     4,eError,# 18-1f
+     4,eError,     4,     4,     4,eError,     4,eError,# 20-27
+     4,eItsMe,eStart,eStart,eStart,eStart,eStart,eStart,# 28-2f
+)
+
+HZCharLenTable = (0, 0, 0, 0, 0, 0)
+
+HZSMModel = {'classTable': HZ_cls,
+             'classFactor': 6,
+             'stateTable': HZ_st,
+             'charLenTable': HZCharLenTable,
+             'name': "HZ-GB-2312"}
+
+ISO2022CN_cls = (
+2,0,0,0,0,0,0,0,  # 00 - 07
+0,0,0,0,0,0,0,0,  # 08 - 0f
+0,0,0,0,0,0,0,0,  # 10 - 17
+0,0,0,1,0,0,0,0,  # 18 - 1f
+0,0,0,0,0,0,0,0,  # 20 - 27
+0,3,0,0,0,0,0,0,  # 28 - 2f
+0,0,0,0,0,0,0,0,  # 30 - 37
+0,0,0,0,0,0,0,0,  # 38 - 3f
+0,0,0,4,0,0,0,0,  # 40 - 47
+0,0,0,0,0,0,0,0,  # 48 - 4f
+0,0,0,0,0,0,0,0,  # 50 - 57
+0,0,0,0,0,0,0,0,  # 58 - 5f
+0,0,0,0,0,0,0,0,  # 60 - 67
+0,0,0,0,0,0,0,0,  # 68 - 6f
+0,0,0,0,0,0,0,0,  # 70 - 77
+0,0,0,0,0,0,0,0,  # 78 - 7f
+2,2,2,2,2,2,2,2,  # 80 - 87
+2,2,2,2,2,2,2,2,  # 88 - 8f
+2,2,2,2,2,2,2,2,  # 90 - 97
+2,2,2,2,2,2,2,2,  # 98 - 9f
+2,2,2,2,2,2,2,2,  # a0 - a7
+2,2,2,2,2,2,2,2,  # a8 - af
+2,2,2,2,2,2,2,2,  # b0 - b7
+2,2,2,2,2,2,2,2,  # b8 - bf
+2,2,2,2,2,2,2,2,  # c0 - c7
+2,2,2,2,2,2,2,2,  # c8 - cf
+2,2,2,2,2,2,2,2,  # d0 - d7
+2,2,2,2,2,2,2,2,  # d8 - df
+2,2,2,2,2,2,2,2,  # e0 - e7
+2,2,2,2,2,2,2,2,  # e8 - ef
+2,2,2,2,2,2,2,2,  # f0 - f7
+2,2,2,2,2,2,2,2,  # f8 - ff
+)
+
+ISO2022CN_st = (
+eStart,     3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07
+eStart,eError,eError,eError,eError,eError,eError,eError,# 08-0f
+eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17
+eItsMe,eItsMe,eItsMe,eError,eError,eError,     4,eError,# 18-1f
+eError,eError,eError,eItsMe,eError,eError,eError,eError,# 20-27
+     5,     6,eError,eError,eError,eError,eError,eError,# 28-2f
+eError,eError,eError,eItsMe,eError,eError,eError,eError,# 30-37
+eError,eError,eError,eError,eError,eItsMe,eError,eStart,# 38-3f
+)
+
+ISO2022CNCharLenTable = (0, 0, 0, 0, 0, 0, 0, 0, 0)
+
+ISO2022CNSMModel = {'classTable': ISO2022CN_cls,
+                    'classFactor': 9,
+                    'stateTable': ISO2022CN_st,
+                    'charLenTable': ISO2022CNCharLenTable,
+                    'name': "ISO-2022-CN"}
+
+ISO2022JP_cls = (
+2,0,0,0,0,0,0,0,  # 00 - 07
+0,0,0,0,0,0,2,2,  # 08 - 0f
+0,0,0,0,0,0,0,0,  # 10 - 17
+0,0,0,1,0,0,0,0,  # 18 - 1f
+0,0,0,0,7,0,0,0,  # 20 - 27
+3,0,0,0,0,0,0,0,  # 28 - 2f
+0,0,0,0,0,0,0,0,  # 30 - 37
+0,0,0,0,0,0,0,0,  # 38 - 3f
+6,0,4,0,8,0,0,0,  # 40 - 47
+0,9,5,0,0,0,0,0,  # 48 - 4f
+0,0,0,0,0,0,0,0,  # 50 - 57
+0,0,0,0,0,0,0,0,  # 58 - 5f
+0,0,0,0,0,0,0,0,  # 60 - 67
+0,0,0,0,0,0,0,0,  # 68 - 6f
+0,0,0,0,0,0,0,0,  # 70 - 77
+0,0,0,0,0,0,0,0,  # 78 - 7f
+2,2,2,2,2,2,2,2,  # 80 - 87
+2,2,2,2,2,2,2,2,  # 88 - 8f
+2,2,2,2,2,2,2,2,  # 90 - 97
+2,2,2,2,2,2,2,2,  # 98 - 9f
+2,2,2,2,2,2,2,2,  # a0 - a7
+2,2,2,2,2,2,2,2,  # a8 - af
+2,2,2,2,2,2,2,2,  # b0 - b7
+2,2,2,2,2,2,2,2,  # b8 - bf
+2,2,2,2,2,2,2,2,  # c0 - c7
+2,2,2,2,2,2,2,2,  # c8 - cf
+2,2,2,2,2,2,2,2,  # d0 - d7
+2,2,2,2,2,2,2,2,  # d8 - df
+2,2,2,2,2,2,2,2,  # e0 - e7
+2,2,2,2,2,2,2,2,  # e8 - ef
+2,2,2,2,2,2,2,2,  # f0 - f7
+2,2,2,2,2,2,2,2,  # f8 - ff
+)
+
+ISO2022JP_st = (
+eStart,     3,eError,eStart,eStart,eStart,eStart,eStart,# 00-07
+eStart,eStart,eError,eError,eError,eError,eError,eError,# 08-0f
+eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 10-17
+eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,# 18-1f
+eError,     5,eError,eError,eError,     4,eError,eError,# 20-27
+eError,eError,eError,     6,eItsMe,eError,eItsMe,eError,# 28-2f
+eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,# 30-37
+eError,eError,eError,eItsMe,eError,eError,eError,eError,# 38-3f
+eError,eError,eError,eError,eItsMe,eError,eStart,eStart,# 40-47
+)
+
+ISO2022JPCharLenTable = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+
+ISO2022JPSMModel = {'classTable': ISO2022JP_cls,
+                    'classFactor': 10,
+                    'stateTable': ISO2022JP_st,
+                    'charLenTable': ISO2022JPCharLenTable,
+                    'name': "ISO-2022-JP"}
+
+ISO2022KR_cls = (
+2,0,0,0,0,0,0,0,  # 00 - 07
+0,0,0,0,0,0,0,0,  # 08 - 0f
+0,0,0,0,0,0,0,0,  # 10 - 17
+0,0,0,1,0,0,0,0,  # 18 - 1f
+0,0,0,0,3,0,0,0,  # 20 - 27
+0,4,0,0,0,0,0,0,  # 28 - 2f
+0,0,0,0,0,0,0,0,  # 30 - 37
+0,0,0,0,0,0,0,0,  # 38 - 3f
+0,0,0,5,0,0,0,0,  # 40 - 47
+0,0,0,0,0,0,0,0,  # 48 - 4f
+0,0,0,0,0,0,0,0,  # 50 - 57
+0,0,0,0,0,0,0,0,  # 58 - 5f
+0,0,0,0,0,0,0,0,  # 60 - 67
+0,0,0,0,0,0,0,0,  # 68 - 6f
+0,0,0,0,0,0,0,0,  # 70 - 77
+0,0,0,0,0,0,0,0,  # 78 - 7f
+2,2,2,2,2,2,2,2,  # 80 - 87
+2,2,2,2,2,2,2,2,  # 88 - 8f
+2,2,2,2,2,2,2,2,  # 90 - 97
+2,2,2,2,2,2,2,2,  # 98 - 9f
+2,2,2,2,2,2,2,2,  # a0 - a7
+2,2,2,2,2,2,2,2,  # a8 - af
+2,2,2,2,2,2,2,2,  # b0 - b7
+2,2,2,2,2,2,2,2,  # b8 - bf
+2,2,2,2,2,2,2,2,  # c0 - c7
+2,2,2,2,2,2,2,2,  # c8 - cf
+2,2,2,2,2,2,2,2,  # d0 - d7
+2,2,2,2,2,2,2,2,  # d8 - df
+2,2,2,2,2,2,2,2,  # e0 - e7
+2,2,2,2,2,2,2,2,  # e8 - ef
+2,2,2,2,2,2,2,2,  # f0 - f7
+2,2,2,2,2,2,2,2,  # f8 - ff
+)
+
+ISO2022KR_st = (
+eStart,     3,eError,eStart,eStart,eStart,eError,eError,# 00-07
+eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,# 08-0f
+eItsMe,eItsMe,eError,eError,eError,     4,eError,eError,# 10-17
+eError,eError,eError,eError,     5,eError,eError,eError,# 18-1f
+eError,eError,eError,eItsMe,eStart,eStart,eStart,eStart,# 20-27
+)
+
+ISO2022KRCharLenTable = (0, 0, 0, 0, 0, 0)
+
+ISO2022KRSMModel = {'classTable': ISO2022KR_cls,
+                    'classFactor': 6,
+                    'stateTable': ISO2022KR_st,
+                    'charLenTable': ISO2022KRCharLenTable,
+                    'name': "ISO-2022-KR"}
+
+# flake8: noqa

chardet/eucjpprober.py

@@ -13,25 +13,26 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, sys
-from constants import eStart, eError, eItsMe
-from mbcharsetprober import MultiByteCharSetProber
-from codingstatemachine import CodingStateMachine
-from chardistribution import EUCJPDistributionAnalysis
-from jpcntx import EUCJPContextAnalysis
-from mbcssm import EUCJPSMModel
+import sys
+from . import constants
+from .mbcharsetprober import MultiByteCharSetProber
+from .codingstatemachine import CodingStateMachine
+from .chardistribution import EUCJPDistributionAnalysis
+from .jpcntx import EUCJPContextAnalysis
+from .mbcssm import EUCJPSMModel
+
 
 class EUCJPProber(MultiByteCharSetProber):
     def __init__(self):
@@ -44,37 +45,41 @@ class EUCJPProber(MultiByteCharSetProber):
     def reset(self):
         MultiByteCharSetProber.reset(self)
         self._mContextAnalyzer.reset()
-        
+
     def get_charset_name(self):
         return "EUC-JP"
 
     def feed(self, aBuf):
         aLen = len(aBuf)
         for i in range(0, aLen):
+            # PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte
             codingState = self._mCodingSM.next_state(aBuf[i])
-            if codingState == eError:
+            if codingState == constants.eError:
                 if constants._debug:
-                    sys.stderr.write(self.get_charset_name() + ' prober hit error at byte ' + str(i) + '\n')
+                    sys.stderr.write(self.get_charset_name()
+                                     + ' prober hit error at byte ' + str(i)
+                                     + '\n')
                 self._mState = constants.eNotMe
                 break
-            elif codingState == eItsMe:
+            elif codingState == constants.eItsMe:
                 self._mState = constants.eFoundIt
                 break
-            elif codingState == eStart:
+            elif codingState == constants.eStart:
                 charLen = self._mCodingSM.get_current_charlen()
                 if i == 0:
                     self._mLastChar[1] = aBuf[0]
                     self._mContextAnalyzer.feed(self._mLastChar, charLen)
                     self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
                 else:
-                    self._mContextAnalyzer.feed(aBuf[i-1:i+1], charLen)
-                    self._mDistributionAnalyzer.feed(aBuf[i-1:i+1], charLen)
-                    
+                    self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen)
+                    self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
+                                                     charLen)
+
         self._mLastChar[0] = aBuf[aLen - 1]
-        
+
         if self.get_state() == constants.eDetecting:
-            if self._mContextAnalyzer.got_enough_data() and \
-                   (self.get_confidence() > constants.SHORTCUT_THRESHOLD):
+            if (self._mContextAnalyzer.got_enough_data() and
+               (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
                 self._mState = constants.eFoundIt
 
         return self.get_state()

chardet/euckrfreq.py

@@ -592,3 +592,5 @@ EUCKRCharToFreqOrder = ( \
 8704,8705,8706,8707,8708,8709,8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,
 8720,8721,8722,8723,8724,8725,8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,
 8736,8737,8738,8739,8740,8741)
+
+# flake8: noqa

chardet/euckrprober.py

@@ -13,22 +13,23 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-from mbcharsetprober import MultiByteCharSetProber
-from codingstatemachine import CodingStateMachine
-from chardistribution import EUCKRDistributionAnalysis
-from mbcssm import EUCKRSMModel
+from .mbcharsetprober import MultiByteCharSetProber
+from .codingstatemachine import CodingStateMachine
+from .chardistribution import EUCKRDistributionAnalysis
+from .mbcssm import EUCKRSMModel
+
 
 class EUCKRProber(MultiByteCharSetProber):
     def __init__(self):

chardet/euctwfreq.py

@@ -13,12 +13,12 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
@@ -26,8 +26,8 @@
 ######################### END LICENSE BLOCK #########################
 
 # EUCTW frequency table
-# Converted from big5 work 
-# by Taiwan's Mandarin Promotion Council 
+# Converted from big5 work
+# by Taiwan's Mandarin Promotion Council
 # <http:#www.edu.tw:81/mandr/>
 
 # 128  --> 0.42261
@@ -38,15 +38,15 @@
 #
 # Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98
 # Random Distribution Ration = 512/(5401-512)=0.105
-# 
+#
 # Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR
 
 EUCTW_TYPICAL_DISTRIBUTION_RATIO = 0.75
 
-# Char to FreqOrder table , 
+# Char to FreqOrder table ,
 EUCTW_TABLE_SIZE = 8102
 
-EUCTWCharToFreqOrder = ( \
+EUCTWCharToFreqOrder = (
    1,1800,1506, 255,1431, 198,   9,  82,   6,7310, 177, 202,3615,1256,2808, 110, # 2742
 3735,  33,3241, 261,  76,  44,2113,  16,2931,2184,1176, 659,3868,  26,3404,2643, # 2758
 1198,3869,3313,4060, 410,2211, 302, 590, 361,1963,   8, 204,  58,4296,7311,1931, # 2774
@@ -424,3 +424,5 @@ EUCTWCharToFreqOrder = ( \
 8694,8695,8696,8697,8698,8699,8700,8701,8702,8703,8704,8705,8706,8707,8708,8709, # 8710
 8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,8720,8721,8722,8723,8724,8725, # 8726
 8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,8736,8737,8738,8739,8740,8741) # 8742
+
+# flake8: noqa

chardet/euctwprober.py

@@ -25,10 +25,10 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-from mbcharsetprober import MultiByteCharSetProber
-from codingstatemachine import CodingStateMachine
-from chardistribution import EUCTWDistributionAnalysis
-from mbcssm import EUCTWSMModel
+from .mbcharsetprober import MultiByteCharSetProber
+from .codingstatemachine import CodingStateMachine
+from .chardistribution import EUCTWDistributionAnalysis
+from .mbcssm import EUCTWSMModel
 
 class EUCTWProber(MultiByteCharSetProber):
     def __init__(self):

chardet/gb2312freq.py

@@ -13,12 +13,12 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
@@ -36,14 +36,14 @@
 #
 # Ideal Distribution Ratio = 0.79135/(1-0.79135) = 3.79
 # Random Distribution Ration = 512 / (3755 - 512) = 0.157
-# 
+#
 # Typical Distribution Ratio about 25% of Ideal one, still much higher that RDR
 
 GB2312_TYPICAL_DISTRIBUTION_RATIO = 0.9
 
 GB2312_TABLE_SIZE = 3760
 
-GB2312CharToFreqOrder = ( \
+GB2312CharToFreqOrder = (
 1671, 749,1443,2364,3924,3807,2330,3921,1704,3463,2691,1511,1515, 572,3191,2205,
 2361, 224,2558, 479,1711, 963,3162, 440,4060,1905,2966,2947,3580,2647,3961,3842,
 2204, 869,4207, 970,2678,5626,2944,2956,1479,4048, 514,3595, 588,1346,2820,3409,
@@ -469,3 +469,4 @@ GB2312CharToFreqOrder = ( \
 5867,5507,6273,4206,6274,4789,6098,6764,3619,3646,3833,3804,2394,3788,4936,3978,
 4866,4899,6099,6100,5559,6478,6765,3599,5868,6101,5869,5870,6275,6766,4527,6767)
 
+# flake8: noqa

chardet/gb2312prober.py

@@ -25,10 +25,10 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-from mbcharsetprober import MultiByteCharSetProber
-from codingstatemachine import CodingStateMachine
-from chardistribution import GB2312DistributionAnalysis
-from mbcssm import GB2312SMModel
+from .mbcharsetprober import MultiByteCharSetProber
+from .codingstatemachine import CodingStateMachine
+from .chardistribution import GB2312DistributionAnalysis
+from .mbcssm import GB2312SMModel
 
 class GB2312Prober(MultiByteCharSetProber):
     def __init__(self):

chardet/hebrewprober.py

@@ -13,20 +13,21 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-from charsetprober import CharSetProber
-import constants
+from .charsetprober import CharSetProber
+from .constants import eNotMe, eDetecting
+from .compat import wrap_ord
 
 # This prober doesn't actually recognize a language or a charset.
 # It is a helper prober for the use of the Hebrew model probers
@@ -35,40 +36,40 @@ import constants
 #
 # Four main charsets exist in Hebrew:
 # "ISO-8859-8" - Visual Hebrew
-# "windows-1255" - Logical Hebrew 
+# "windows-1255" - Logical Hebrew
 # "ISO-8859-8-I" - Logical Hebrew
 # "x-mac-hebrew" - ?? Logical Hebrew ??
 #
 # Both "ISO" charsets use a completely identical set of code points, whereas
-# "windows-1255" and "x-mac-hebrew" are two different proper supersets of 
+# "windows-1255" and "x-mac-hebrew" are two different proper supersets of
 # these code points. windows-1255 defines additional characters in the range
-# 0x80-0x9F as some misc punctuation marks as well as some Hebrew-specific 
+# 0x80-0x9F as some misc punctuation marks as well as some Hebrew-specific
 # diacritics and additional 'Yiddish' ligature letters in the range 0xc0-0xd6.
-# x-mac-hebrew defines similar additional code points but with a different 
+# x-mac-hebrew defines similar additional code points but with a different
 # mapping.
 #
-# As far as an average Hebrew text with no diacritics is concerned, all four 
-# charsets are identical with respect to code points. Meaning that for the 
-# main Hebrew alphabet, all four map the same values to all 27 Hebrew letters 
+# As far as an average Hebrew text with no diacritics is concerned, all four
+# charsets are identical with respect to code points. Meaning that for the
+# main Hebrew alphabet, all four map the same values to all 27 Hebrew letters
 # (including final letters).
 #
 # The dominant difference between these charsets is their directionality.
 # "Visual" directionality means that the text is ordered as if the renderer is
-# not aware of a BIDI rendering algorithm. The renderer sees the text and 
-# draws it from left to right. The text itself when ordered naturally is read 
+# not aware of a BIDI rendering algorithm. The renderer sees the text and
+# draws it from left to right. The text itself when ordered naturally is read
 # backwards. A buffer of Visual Hebrew generally looks like so:
 # "[last word of first line spelled backwards] [whole line ordered backwards
-# and spelled backwards] [first word of first line spelled backwards] 
+# and spelled backwards] [first word of first line spelled backwards]
 # [end of line] [last word of second line] ... etc' "
 # adding punctuation marks, numbers and English text to visual text is
 # naturally also "visual" and from left to right.
-# 
+#
 # "Logical" directionality means the text is ordered "naturally" according to
-# the order it is read. It is the responsibility of the renderer to display 
-# the text from right to left. A BIDI algorithm is used to place general 
+# the order it is read. It is the responsibility of the renderer to display
+# the text from right to left. A BIDI algorithm is used to place general
 # punctuation marks, numbers and English text in the text.
 #
-# Texts in x-mac-hebrew are almost impossible to find on the Internet. From 
+# Texts in x-mac-hebrew are almost impossible to find on the Internet. From
 # what little evidence I could find, it seems that its general directionality
 # is Logical.
 #
@@ -76,17 +77,17 @@ import constants
 # charsets:
 # Visual Hebrew - "ISO-8859-8" - backwards text - Words and sentences are
 #    backwards while line order is natural. For charset recognition purposes
-#    the line order is unimportant (In fact, for this implementation, even 
+#    the line order is unimportant (In fact, for this implementation, even
 #    word order is unimportant).
 # Logical Hebrew - "windows-1255" - normal, naturally ordered text.
 #
-# "ISO-8859-8-I" is a subset of windows-1255 and doesn't need to be 
+# "ISO-8859-8-I" is a subset of windows-1255 and doesn't need to be
 #    specifically identified.
 # "x-mac-hebrew" is also identified as windows-1255. A text in x-mac-hebrew
 #    that contain special punctuation marks or diacritics is displayed with
 #    some unconverted characters showing as question marks. This problem might
 #    be corrected using another model prober for x-mac-hebrew. Due to the fact
-#    that x-mac-hebrew texts are so rare, writing another model prober isn't 
+#    that x-mac-hebrew texts are so rare, writing another model prober isn't
 #    worth the effort and performance hit.
 #
 #### The Prober ####
@@ -126,28 +127,31 @@ import constants
 # charset identified, either "windows-1255" or "ISO-8859-8".
 
 # windows-1255 / ISO-8859-8 code points of interest
-FINAL_KAF = '\xea'
-NORMAL_KAF = '\xeb'
-FINAL_MEM = '\xed'
-NORMAL_MEM = '\xee'
-FINAL_NUN = '\xef'
-NORMAL_NUN = '\xf0'
-FINAL_PE = '\xf3'
-NORMAL_PE = '\xf4'
-FINAL_TSADI = '\xf5'
-NORMAL_TSADI = '\xf6'
+FINAL_KAF = 0xea
+NORMAL_KAF = 0xeb
+FINAL_MEM = 0xed
+NORMAL_MEM = 0xee
+FINAL_NUN = 0xef
+NORMAL_NUN = 0xf0
+FINAL_PE = 0xf3
+NORMAL_PE = 0xf4
+FINAL_TSADI = 0xf5
+NORMAL_TSADI = 0xf6
 
 # Minimum Visual vs Logical final letter score difference.
-# If the difference is below this, don't rely solely on the final letter score distance.
+# If the difference is below this, don't rely solely on the final letter score
+# distance.
 MIN_FINAL_CHAR_DISTANCE = 5
 
 # Minimum Visual vs Logical model score difference.
-# If the difference is below this, don't rely at all on the model score distance.
+# If the difference is below this, don't rely at all on the model score
+# distance.
 MIN_MODEL_DISTANCE = 0.01
 
 VISUAL_HEBREW_NAME = "ISO-8859-8"
 LOGICAL_HEBREW_NAME = "windows-1255"
 
+
 class HebrewProber(CharSetProber):
     def __init__(self):
         CharSetProber.__init__(self)
@@ -159,84 +163,91 @@ class HebrewProber(CharSetProber):
         self._mFinalCharLogicalScore = 0
         self._mFinalCharVisualScore = 0
         # The two last characters seen in the previous buffer,
-        # mPrev and mBeforePrev are initialized to space in order to simulate a word 
-        # delimiter at the beginning of the data
+        # mPrev and mBeforePrev are initialized to space in order to simulate
+        # a word delimiter at the beginning of the data
         self._mPrev = ' '
         self._mBeforePrev = ' '
         # These probers are owned by the group prober.
-        
+
     def set_model_probers(self, logicalProber, visualProber):
         self._mLogicalProber = logicalProber
         self._mVisualProber = visualProber
 
     def is_final(self, c):
-        return c in [FINAL_KAF, FINAL_MEM, FINAL_NUN, FINAL_PE, FINAL_TSADI]
+        return wrap_ord(c) in [FINAL_KAF, FINAL_MEM, FINAL_NUN, FINAL_PE,
+                               FINAL_TSADI]
 
     def is_non_final(self, c):
-        # The normal Tsadi is not a good Non-Final letter due to words like 
-        # 'lechotet' (to chat) containing an apostrophe after the tsadi. This 
-        # apostrophe is converted to a space in FilterWithoutEnglishLetters causing 
-        # the Non-Final tsadi to appear at an end of a word even though this is not 
-        # the case in the original text.
-        # The letters Pe and Kaf rarely display a related behavior of not being a 
-        # good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak' for 
-        # example legally end with a Non-Final Pe or Kaf. However, the benefit of 
-        # these letters as Non-Final letters outweighs the damage since these words 
-        # are quite rare.
-        return c in [NORMAL_KAF, NORMAL_MEM, NORMAL_NUN, NORMAL_PE]
-    
+        # The normal Tsadi is not a good Non-Final letter due to words like
+        # 'lechotet' (to chat) containing an apostrophe after the tsadi. This
+        # apostrophe is converted to a space in FilterWithoutEnglishLetters
+        # causing the Non-Final tsadi to appear at an end of a word even
+        # though this is not the case in the original text.
+        # The letters Pe and Kaf rarely display a related behavior of not being
+        # a good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak'
+        # for example legally end with a Non-Final Pe or Kaf. However, the
+        # benefit of these letters as Non-Final letters outweighs the damage
+        # since these words are quite rare.
+        return wrap_ord(c) in [NORMAL_KAF, NORMAL_MEM, NORMAL_NUN, NORMAL_PE]
+
     def feed(self, aBuf):
         # Final letter analysis for logical-visual decision.
-        # Look for evidence that the received buffer is either logical Hebrew or 
-        # visual Hebrew.
+        # Look for evidence that the received buffer is either logical Hebrew
+        # or visual Hebrew.
         # The following cases are checked:
-        # 1) A word longer than 1 letter, ending with a final letter. This is an 
-        #    indication that the text is laid out "naturally" since the final letter 
-        #    really appears at the end. +1 for logical score.
-        # 2) A word longer than 1 letter, ending with a Non-Final letter. In normal
-        #    Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi, should not end with
-        #    the Non-Final form of that letter. Exceptions to this rule are mentioned
-        #    above in isNonFinal(). This is an indication that the text is laid out
-        #    backwards. +1 for visual score
-        # 3) A word longer than 1 letter, starting with a final letter. Final letters 
-        #    should not appear at the beginning of a word. This is an indication that 
-        #    the text is laid out backwards. +1 for visual score.
-        # 
-        # The visual score and logical score are accumulated throughout the text and 
-        # are finally checked against each other in GetCharSetName().
-        # No checking for final letters in the middle of words is done since that case
-        # is not an indication for either Logical or Visual text.
-        # 
-        # We automatically filter out all 7-bit characters (replace them with spaces)
-        # so the word boundary detection works properly. [MAP]
+        # 1) A word longer than 1 letter, ending with a final letter. This is
+        #    an indication that the text is laid out "naturally" since the
+        #    final letter really appears at the end. +1 for logical score.
+        # 2) A word longer than 1 letter, ending with a Non-Final letter. In
+        #    normal Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi,
+        #    should not end with the Non-Final form of that letter. Exceptions
+        #    to this rule are mentioned above in isNonFinal(). This is an
+        #    indication that the text is laid out backwards. +1 for visual
+        #    score
+        # 3) A word longer than 1 letter, starting with a final letter. Final
+        #    letters should not appear at the beginning of a word. This is an
+        #    indication that the text is laid out backwards. +1 for visual
+        #    score.
+        #
+        # The visual score and logical score are accumulated throughout the
+        # text and are finally checked against each other in GetCharSetName().
+        # No checking for final letters in the middle of words is done since
+        # that case is not an indication for either Logical or Visual text.
+        #
+        # We automatically filter out all 7-bit characters (replace them with
+        # spaces) so the word boundary detection works properly. [MAP]
 
-        if self.get_state() == constants.eNotMe:
+        if self.get_state() == eNotMe:
             # Both model probers say it's not them. No reason to continue.
-            return constants.eNotMe
+            return eNotMe
 
         aBuf = self.filter_high_bit_only(aBuf)
-        
+
         for cur in aBuf:
             if cur == ' ':
                 # We stand on a space - a word just ended
                 if self._mBeforePrev != ' ':
-                    # next-to-last char was not a space so self._mPrev is not a 1 letter word
+                    # next-to-last char was not a space so self._mPrev is not a
+                    # 1 letter word
                     if self.is_final(self._mPrev):
                         # case (1) [-2:not space][-1:final letter][cur:space]
                         self._mFinalCharLogicalScore += 1
                     elif self.is_non_final(self._mPrev):
-                        # case (2) [-2:not space][-1:Non-Final letter][cur:space]
+                        # case (2) [-2:not space][-1:Non-Final letter][
+                        #  cur:space]
                         self._mFinalCharVisualScore += 1
             else:
                 # Not standing on a space
-                if (self._mBeforePrev == ' ') and (self.is_final(self._mPrev)) and (cur != ' '):
+                if ((self._mBeforePrev == ' ') and
+                        (self.is_final(self._mPrev)) and (cur != ' ')):
                     # case (3) [-2:space][-1:final letter][cur:not space]
                     self._mFinalCharVisualScore += 1
             self._mBeforePrev = self._mPrev
             self._mPrev = cur
 
-        # Forever detecting, till the end or until both model probers return eNotMe (handled above)
-        return constants.eDetecting
+        # Forever detecting, till the end or until both model probers return
+        # eNotMe (handled above)
+        return eDetecting
 
     def get_charset_name(self):
         # Make the decision: is it Logical or Visual?
@@ -248,22 +259,25 @@ class HebrewProber(CharSetProber):
             return VISUAL_HEBREW_NAME
 
         # It's not dominant enough, try to rely on the model scores instead.
-        modelsub = self._mLogicalProber.get_confidence() - self._mVisualProber.get_confidence()
+        modelsub = (self._mLogicalProber.get_confidence()
+                    - self._mVisualProber.get_confidence())
         if modelsub > MIN_MODEL_DISTANCE:
             return LOGICAL_HEBREW_NAME
         if modelsub < -MIN_MODEL_DISTANCE:
             return VISUAL_HEBREW_NAME
 
-        # Still no good, back to final letter distance, maybe it'll save the day.
+        # Still no good, back to final letter distance, maybe it'll save the
+        # day.
         if finalsub < 0.0:
             return VISUAL_HEBREW_NAME
 
-        # (finalsub > 0 - Logical) or (don't know what to do) default to Logical.
+        # (finalsub > 0 - Logical) or (don't know what to do) default to
+        # Logical.
         return LOGICAL_HEBREW_NAME
 
     def get_state(self):
         # Remain active as long as any of the model probers are active.
-        if (self._mLogicalProber.get_state() == constants.eNotMe) and \
-           (self._mVisualProber.get_state() == constants.eNotMe):
-            return constants.eNotMe
-        return constants.eDetecting
+        if (self._mLogicalProber.get_state() == eNotMe) and \
+           (self._mVisualProber.get_state() == eNotMe):
+            return eNotMe
+        return eDetecting

chardet/jisfreq.py

@@ -13,12 +13,12 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
@@ -28,7 +28,7 @@
 # Sampling from about 20M text materials include literature and computer technology
 #
 # Japanese frequency table, applied to both S-JIS and EUC-JP
-# They are sorted in order. 
+# They are sorted in order.
 
 # 128  --> 0.77094
 # 256  --> 0.85710
@@ -38,15 +38,15 @@
 #
 # Ideal Distribution Ratio = 0.92635 / (1-0.92635) = 12.58
 # Random Distribution Ration = 512 / (2965+62+83+86-512) = 0.191
-# 
-# Typical Distribution Ratio, 25% of IDR 
+#
+# Typical Distribution Ratio, 25% of IDR
 
 JIS_TYPICAL_DISTRIBUTION_RATIO = 3.0
 
-# Char to FreqOrder table , 
+# Char to FreqOrder table ,
 JIS_TABLE_SIZE = 4368
 
-JISCharToFreqOrder = ( \
+JISCharToFreqOrder = (
   40,   1,   6, 182, 152, 180, 295,2127, 285, 381,3295,4304,3068,4606,3165,3510, #   16
 3511,1822,2785,4607,1193,2226,5070,4608, 171,2996,1247,  18, 179,5071, 856,1661, #   32
 1262,5072, 619, 127,3431,3512,3230,1899,1700, 232, 228,1294,1298, 284, 283,2041, #   48
@@ -565,3 +565,5 @@ JISCharToFreqOrder = ( \
 8224,8225,8226,8227,8228,8229,8230,8231,8232,8233,8234,8235,8236,8237,8238,8239, # 8240
 8240,8241,8242,8243,8244,8245,8246,8247,8248,8249,8250,8251,8252,8253,8254,8255, # 8256
 8256,8257,8258,8259,8260,8261,8262,8263,8264,8265,8266,8267,8268,8269,8270,8271) # 8272
+
+# flake8: noqa

chardet/jpcntx.py

@@ -13,19 +13,19 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants
+from .compat import wrap_ord
 
 NUM_OF_CATEGORY = 6
 DONT_KNOW = -1
@@ -34,7 +34,7 @@ MAX_REL_THRESHOLD = 1000
 MINIMUM_DATA_THRESHOLD = 4
 
 # This is hiragana 2-char sequence table, the number in each cell represents its frequency category
-jp2CharContext = ( \
+jp2CharContext = (
 (0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1),
 (2,4,0,4,0,3,0,4,0,3,4,4,4,2,4,3,3,4,3,2,3,3,4,2,3,3,3,2,4,1,4,3,3,1,5,4,3,4,3,4,3,5,3,0,3,5,4,2,0,3,1,0,3,3,0,3,3,0,1,1,0,4,3,0,3,3,0,4,0,2,0,3,5,5,5,5,4,0,4,1,0,3,4),
 (0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2),
@@ -123,26 +123,33 @@ jp2CharContext = ( \
 class JapaneseContextAnalysis:
     def __init__(self):
         self.reset()
-        
+
     def reset(self):
-        self._mTotalRel = 0 # total sequence received
-        self._mRelSample = [0] * NUM_OF_CATEGORY # category counters, each interger counts sequence in its category
-        self._mNeedToSkipCharNum = 0 # if last byte in current buffer is not the last byte of a character, we need to know how many bytes to skip in next buffer
-        self._mLastCharOrder = -1 # The order of previous char
-        self._mDone = constants.False # If this flag is set to constants.True, detection is done and conclusion has been made
+        self._mTotalRel = 0  # total sequence received
+        # category counters, each interger counts sequence in its category
+        self._mRelSample = [0] * NUM_OF_CATEGORY
+        # if last byte in current buffer is not the last byte of a character,
+        # we need to know how many bytes to skip in next buffer
+        self._mNeedToSkipCharNum = 0
+        self._mLastCharOrder = -1  # The order of previous char
+        # If this flag is set to True, detection is done and conclusion has
+        # been made
+        self._mDone = False
 
     def feed(self, aBuf, aLen):
-        if self._mDone: return
-        
+        if self._mDone:
+            return
+
         # The buffer we got is byte oriented, and a character may span in more than one
-        # buffers. In case the last one or two byte in last buffer is not complete, we 
-        # record how many byte needed to complete that character and skip these bytes here.
-        # We can choose to record those bytes as well and analyse the character once it 
-        # is complete, but since a character will not make much difference, by simply skipping
+        # buffers. In case the last one or two byte in last buffer is not
+        # complete, we record how many byte needed to complete that character
+        # and skip these bytes here.  We can choose to record those bytes as
+        # well and analyse the character once it is complete, but since a
+        # character will not make much difference, by simply skipping
         # this character will simply our logic and improve performance.
         i = self._mNeedToSkipCharNum
         while i < aLen:
-            order, charLen = self.get_order(aBuf[i:i+2])
+            order, charLen = self.get_order(aBuf[i:i + 2])
             i += charLen
             if i > aLen:
                 self._mNeedToSkipCharNum = i - aLen
@@ -151,14 +158,14 @@ class JapaneseContextAnalysis:
                 if (order != -1) and (self._mLastCharOrder != -1):
                     self._mTotalRel += 1
                     if self._mTotalRel > MAX_REL_THRESHOLD:
-                        self._mDone = constants.True
+                        self._mDone = True
                         break
                     self._mRelSample[jp2CharContext[self._mLastCharOrder][order]] += 1
                 self._mLastCharOrder = order
 
     def got_enough_data(self):
         return self._mTotalRel > ENOUGH_REL_THRESHOLD
-    
+
     def get_confidence(self):
         # This is just one way to calculate confidence. It works well for me.
         if self._mTotalRel > MINIMUM_DATA_THRESHOLD:
@@ -166,45 +173,55 @@ class JapaneseContextAnalysis:
         else:
             return DONT_KNOW
 
-    def get_order(self, aStr):
+    def get_order(self, aBuf):
         return -1, 1
-        
+
 class SJISContextAnalysis(JapaneseContextAnalysis):
-    def get_order(self, aStr):
-        if not aStr: return -1, 1
+    def __init__(self):
+        self.charset_name = "SHIFT_JIS"
+
+    def get_charset_name(self):
+        return self.charset_name
+
+    def get_order(self, aBuf):
+        if not aBuf:
+            return -1, 1
         # find out current char's byte length
-        if ((aStr[0] >= '\x81') and (aStr[0] <= '\x9F')) or \
-           ((aStr[0] >= '\xE0') and (aStr[0] <= '\xFC')):
+        first_char = wrap_ord(aBuf[0])
+        if ((0x81 <= first_char <= 0x9F) or (0xE0 <= first_char <= 0xFC)):
             charLen = 2
+            if (first_char == 0x87) or (0xFA <= first_char <= 0xFC):
+                self.charset_name = "CP932"
         else:
             charLen = 1
 
         # return its order if it is hiragana
-        if len(aStr) > 1:
-            if (aStr[0] == '\202') and \
-               (aStr[1] >= '\x9F') and \
-               (aStr[1] <= '\xF1'):
-                return ord(aStr[1]) - 0x9F, charLen
+        if len(aBuf) > 1:
+            second_char = wrap_ord(aBuf[1])
+            if (first_char == 202) and (0x9F <= second_char <= 0xF1):
+                return second_char - 0x9F, charLen
 
         return -1, charLen
 
 class EUCJPContextAnalysis(JapaneseContextAnalysis):
-    def get_order(self, aStr):
-        if not aStr: return -1, 1
+    def get_order(self, aBuf):
+        if not aBuf:
+            return -1, 1
         # find out current char's byte length
-        if (aStr[0] == '\x8E') or \
-           ((aStr[0] >= '\xA1') and (aStr[0] <= '\xFE')):
+        first_char = wrap_ord(aBuf[0])
+        if (first_char == 0x8E) or (0xA1 <= first_char <= 0xFE):
             charLen = 2
-        elif aStr[0] == '\x8F':
+        elif first_char == 0x8F:
             charLen = 3
         else:
             charLen = 1
 
         # return its order if it is hiragana
-        if len(aStr) > 1:
-            if (aStr[0] == '\xA4') and \
-               (aStr[1] >= '\xA1') and \
-               (aStr[1] <= '\xF3'):
-                return ord(aStr[1]) - 0xA1, charLen
+        if len(aBuf) > 1:
+            second_char = wrap_ord(aBuf[1])
+            if (first_char == 0xA4) and (0xA1 <= second_char <= 0xF3):
+                return second_char - 0xA1, charLen
 
         return -1, charLen
+
+# flake8: noqa

chardet/langbulgarianmodel.py

@@ -13,30 +13,28 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants
-
 # 255: Control characters that usually does not exist in any text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
 
 # Character Mapping Table:
-# this table is modified base on win1251BulgarianCharToOrderMap, so 
+# this table is modified base on win1251BulgarianCharToOrderMap, so
 # only number <64 is sure valid
 
-Latin5_BulgarianCharToOrderMap = ( \
+Latin5_BulgarianCharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -55,7 +53,7 @@ Latin5_BulgarianCharToOrderMap = ( \
  62,242,243,244, 58,245, 98,246,247,248,249,250,251, 91,252,253,  # f0
 )
 
-win1251BulgarianCharToOrderMap = ( \
+win1251BulgarianCharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -74,13 +72,13 @@ win1251BulgarianCharToOrderMap = ( \
   7,  8,  5, 19, 29, 25, 22, 21, 27, 24, 17, 75, 52,253, 42, 16,  # f0
 )
 
-# Model Table: 
+# Model Table:
 # total sequences: 100%
 # first 512 sequences: 96.9392%
 # first 1024 sequences:3.0618%
 # rest  sequences:     0.2992%
-# negative sequences:  0.0020% 
-BulgarianLangModel = ( \
+# negative sequences:  0.0020%
+BulgarianLangModel = (
 0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,3,3,3,3,3,
 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,2,2,3,2,2,1,2,2,
 3,1,3,3,2,3,3,3,3,3,3,3,3,3,3,3,3,0,3,3,3,3,3,3,3,3,3,3,0,3,0,1,
@@ -211,18 +209,21 @@ BulgarianLangModel = ( \
 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
 )
 
-Latin5BulgarianModel = { \
+Latin5BulgarianModel = {
   'charToOrderMap': Latin5_BulgarianCharToOrderMap,
   'precedenceMatrix': BulgarianLangModel,
   'mTypicalPositiveRatio': 0.969392,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "ISO-8859-5"
 }
 
-Win1251BulgarianModel = { \
+Win1251BulgarianModel = {
   'charToOrderMap': win1251BulgarianCharToOrderMap,
   'precedenceMatrix': BulgarianLangModel,
   'mTypicalPositiveRatio': 0.969392,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "windows-1251"
 }
+
+
+# flake8: noqa

chardet/langcyrillicmodel.py

@@ -13,23 +13,21 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants
-
 # KOI8-R language model
 # Character Mapping Table:
-KOI8R_CharToOrderMap = ( \
+KOI8R_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -48,7 +46,7 @@ KOI8R_CharToOrderMap = ( \
  35, 43, 45, 32, 40, 52, 56, 33, 61, 62, 51, 57, 47, 63, 50, 70,  # f0
 )
 
-win1251_CharToOrderMap = ( \
+win1251_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -67,7 +65,7 @@ win1251_CharToOrderMap = ( \
   9,  7,  6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16,
 )
 
-latin5_CharToOrderMap = ( \
+latin5_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -86,7 +84,7 @@ latin5_CharToOrderMap = ( \
 239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255,
 )
 
-macCyrillic_CharToOrderMap = ( \
+macCyrillic_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -105,7 +103,7 @@ macCyrillic_CharToOrderMap = ( \
   9,  7,  6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27,255,
 )
 
-IBM855_CharToOrderMap = ( \
+IBM855_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -124,7 +122,7 @@ IBM855_CharToOrderMap = ( \
 250, 18, 62, 20, 51, 25, 57, 30, 47, 29, 63, 22, 50,251,252,255,
 )
 
-IBM866_CharToOrderMap = ( \
+IBM866_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -143,13 +141,13 @@ IBM866_CharToOrderMap = ( \
 239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255,
 )
 
-# Model Table: 
+# Model Table:
 # total sequences: 100%
 # first 512 sequences: 97.6601%
 # first 1024 sequences: 2.3389%
 # rest  sequences:      0.1237%
-# negative sequences:   0.0009% 
-RussianLangModel = ( \
+# negative sequences:   0.0009%
+RussianLangModel = (
 0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,3,3,3,3,1,3,3,3,2,3,2,3,3,
 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,2,2,2,2,2,0,0,2,
 3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,3,2,3,2,0,
@@ -280,50 +278,52 @@ RussianLangModel = ( \
 0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,
 )
 
-Koi8rModel = { \
+Koi8rModel = {
   'charToOrderMap': KOI8R_CharToOrderMap,
   'precedenceMatrix': RussianLangModel,
   'mTypicalPositiveRatio': 0.976601,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "KOI8-R"
 }
 
-Win1251CyrillicModel = { \
+Win1251CyrillicModel = {
   'charToOrderMap': win1251_CharToOrderMap,
   'precedenceMatrix': RussianLangModel,
   'mTypicalPositiveRatio': 0.976601,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "windows-1251"
 }
 
-Latin5CyrillicModel = { \
+Latin5CyrillicModel = {
   'charToOrderMap': latin5_CharToOrderMap,
   'precedenceMatrix': RussianLangModel,
   'mTypicalPositiveRatio': 0.976601,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "ISO-8859-5"
 }
 
-MacCyrillicModel = { \
+MacCyrillicModel = {
   'charToOrderMap': macCyrillic_CharToOrderMap,
   'precedenceMatrix': RussianLangModel,
   'mTypicalPositiveRatio': 0.976601,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "MacCyrillic"
 };
 
-Ibm866Model = { \
+Ibm866Model = {
   'charToOrderMap': IBM866_CharToOrderMap,
   'precedenceMatrix': RussianLangModel,
   'mTypicalPositiveRatio': 0.976601,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "IBM866"
 }
 
-Ibm855Model = { \
+Ibm855Model = {
   'charToOrderMap': IBM855_CharToOrderMap,
   'precedenceMatrix': RussianLangModel,
   'mTypicalPositiveRatio': 0.976601,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "IBM855"
 }
+
+# flake8: noqa

chardet/langgreekmodel.py

@@ -13,27 +13,25 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants
-
 # 255: Control characters that usually does not exist in any text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
 
 # Character Mapping Table:
-Latin7_CharToOrderMap = ( \
+Latin7_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -52,7 +50,7 @@ Latin7_CharToOrderMap = ( \
   9,  8, 14,  7,  2, 12, 28, 23, 42, 24, 64, 75, 19, 26, 27,253,  # f0
 )
 
-win1253_CharToOrderMap = ( \
+win1253_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -71,13 +69,13 @@ win1253_CharToOrderMap = ( \
   9,  8, 14,  7,  2, 12, 28, 23, 42, 24, 64, 75, 19, 26, 27,253,  # f0
 )
 
-# Model Table: 
+# Model Table:
 # total sequences: 100%
 # first 512 sequences: 98.2851%
 # first 1024 sequences:1.7001%
 # rest  sequences:     0.0359%
-# negative sequences:  0.0148% 
-GreekLangModel = ( \
+# negative sequences:  0.0148%
+GreekLangModel = (
 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
 0,0,3,2,2,3,3,3,3,3,3,3,3,1,3,3,3,0,2,2,3,3,0,3,0,3,2,0,3,3,3,0,
@@ -208,18 +206,20 @@ GreekLangModel = ( \
 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
 )
 
-Latin7GreekModel = { \
+Latin7GreekModel = {
   'charToOrderMap': Latin7_CharToOrderMap,
   'precedenceMatrix': GreekLangModel,
   'mTypicalPositiveRatio': 0.982851,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "ISO-8859-7"
 }
 
-Win1253GreekModel = { \
+Win1253GreekModel = {
   'charToOrderMap': win1253_CharToOrderMap,
   'precedenceMatrix': GreekLangModel,
   'mTypicalPositiveRatio': 0.982851,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "windows-1253"
 }
+
+# flake8: noqa

chardet/langhebrewmodel.py

@@ -15,20 +15,18 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants
-
 # 255: Control characters that usually does not exist in any text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
@@ -36,7 +34,7 @@ import constants
 
 # Windows-1255 language model
 # Character Mapping Table:
-win1255_CharToOrderMap = ( \
+win1255_CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -55,13 +53,13 @@ win1255_CharToOrderMap = ( \
  12, 19, 13, 26, 18, 27, 21, 17,  7, 10,  5,251,252,128, 96,253,
 )
 
-# Model Table: 
+# Model Table:
 # total sequences: 100%
 # first 512 sequences: 98.4004%
 # first 1024 sequences: 1.5981%
 # rest  sequences:      0.087%
-# negative sequences:   0.0015% 
-HebrewLangModel = ( \
+# negative sequences:   0.0015%
+HebrewLangModel = (
 0,3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,3,3,3,2,3,2,1,2,0,1,0,0,
 3,0,3,1,0,0,1,3,2,0,1,1,2,0,2,2,2,1,1,1,1,2,1,1,1,2,0,0,2,2,0,1,
 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,
@@ -192,10 +190,12 @@ HebrewLangModel = ( \
 0,0,0,0,0,0,0,0,0,0,1,2,1,0,0,0,0,0,1,1,1,1,1,0,1,0,0,0,1,1,0,0,
 )
 
-Win1255HebrewModel = { \
+Win1255HebrewModel = {
   'charToOrderMap': win1255_CharToOrderMap,
   'precedenceMatrix': HebrewLangModel,
   'mTypicalPositiveRatio': 0.984004,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "windows-1255"
 }
+
+# flake8: noqa

chardet/langhungarianmodel.py

@@ -13,27 +13,25 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants
-
 # 255: Control characters that usually does not exist in any text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
 
 # Character Mapping Table:
-Latin2_HungarianCharToOrderMap = ( \
+Latin2_HungarianCharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -52,7 +50,7 @@ Latin2_HungarianCharToOrderMap = ( \
 245,246,247, 25, 73, 42, 24,248,249,250, 31, 56, 29,251,252,253,
 )
 
-win1250HungarianCharToOrderMap = ( \
+win1250HungarianCharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -71,13 +69,13 @@ win1250HungarianCharToOrderMap = ( \
 245,246,247, 25, 74, 42, 24,248,249,250, 31, 56, 29,251,252,253,
 )
 
-# Model Table: 
+# Model Table:
 # total sequences: 100%
 # first 512 sequences: 94.7368%
 # first 1024 sequences:5.2623%
 # rest  sequences:     0.8894%
-# negative sequences:  0.0009% 
-HungarianLangModel = ( \
+# negative sequences:  0.0009%
+HungarianLangModel = (
 0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
 3,3,3,3,3,3,3,3,3,3,2,3,3,3,3,3,3,3,3,2,2,3,3,1,1,2,2,2,2,2,1,2,
 3,2,2,3,3,3,3,3,2,3,3,3,3,3,3,1,2,3,3,3,3,2,3,3,1,1,3,3,0,1,1,1,
@@ -208,18 +206,20 @@ HungarianLangModel = ( \
 0,1,1,1,1,1,1,0,1,1,0,1,0,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,
 )
 
-Latin2HungarianModel = { \
+Latin2HungarianModel = {
   'charToOrderMap': Latin2_HungarianCharToOrderMap,
   'precedenceMatrix': HungarianLangModel,
   'mTypicalPositiveRatio': 0.947368,
-  'keepEnglishLetter': constants.True,
+  'keepEnglishLetter': True,
   'charsetName': "ISO-8859-2"
 }
 
-Win1250HungarianModel = { \
+Win1250HungarianModel = {
   'charToOrderMap': win1250HungarianCharToOrderMap,
   'precedenceMatrix': HungarianLangModel,
   'mTypicalPositiveRatio': 0.947368,
-  'keepEnglishLetter': constants.True,
+  'keepEnglishLetter': True,
   'charsetName': "windows-1250"
 }
+
+# flake8: noqa

chardet/langthaimodel.py

@@ -13,29 +13,27 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants
-
 # 255: Control characters that usually does not exist in any text
 # 254: Carriage/Return
 # 253: symbol (punctuation) that does not belong to word
 # 252: 0 - 9
 
-# The following result for thai was collected from a limited sample (1M). 
+# The following result for thai was collected from a limited sample (1M).
 
 # Character Mapping Table:
-TIS620CharToOrderMap = ( \
+TIS620CharToOrderMap = (
 255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255,  # 00
 255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,  # 10
 253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,  # 20
@@ -54,13 +52,13 @@ TIS620CharToOrderMap = ( \
  68, 56, 59, 65, 69, 60, 70, 80, 71, 87,248,249,250,251,252,253,
 )
 
-# Model Table: 
+# Model Table:
 # total sequences: 100%
 # first 512 sequences: 92.6386%
 # first 1024 sequences:7.3177%
 # rest  sequences:     1.0230%
-# negative sequences:  0.0436% 
-ThaiLangModel = ( \
+# negative sequences:  0.0436%
+ThaiLangModel = (
 0,1,3,3,3,3,0,0,3,3,0,3,3,0,3,3,3,3,3,3,3,3,0,0,3,3,3,0,3,3,3,3,
 0,3,3,0,0,0,1,3,0,3,3,2,3,3,0,1,2,3,3,3,3,0,2,0,2,0,0,3,2,1,2,2,
 3,0,3,3,2,3,0,0,3,3,0,3,3,0,3,3,3,3,3,3,3,3,3,0,3,2,3,0,2,2,2,3,
@@ -191,10 +189,12 @@ ThaiLangModel = ( \
 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
 )
 
-TIS620ThaiModel = { \
+TIS620ThaiModel = {
   'charToOrderMap': TIS620CharToOrderMap,
   'precedenceMatrix': ThaiLangModel,
   'mTypicalPositiveRatio': 0.926386,
-  'keepEnglishLetter': constants.False,
+  'keepEnglishLetter': False,
   'charsetName': "TIS-620"
 }
+
+# flake8: noqa

chardet/latin1prober.py

@@ -0,0 +1,139 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .charsetprober import CharSetProber
+from .constants import eNotMe
+from .compat import wrap_ord
+
+FREQ_CAT_NUM = 4
+
+UDF = 0  # undefined
+OTH = 1  # other
+ASC = 2  # ascii capital letter
+ASS = 3  # ascii small letter
+ACV = 4  # accent capital vowel
+ACO = 5  # accent capital other
+ASV = 6  # accent small vowel
+ASO = 7  # accent small other
+CLASS_NUM = 8  # total classes
+
+Latin1_CharToClass = (
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 00 - 07
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 08 - 0F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 10 - 17
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 18 - 1F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 20 - 27
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 28 - 2F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 30 - 37
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 38 - 3F
+    OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 40 - 47
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 48 - 4F
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 50 - 57
+    ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,   # 58 - 5F
+    OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 60 - 67
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 68 - 6F
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 70 - 77
+    ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,   # 78 - 7F
+    OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH,   # 80 - 87
+    OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF,   # 88 - 8F
+    UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 90 - 97
+    OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO,   # 98 - 9F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A0 - A7
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A8 - AF
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B0 - B7
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B8 - BF
+    ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO,   # C0 - C7
+    ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,   # C8 - CF
+    ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH,   # D0 - D7
+    ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO,   # D8 - DF
+    ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO,   # E0 - E7
+    ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,   # E8 - EF
+    ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH,   # F0 - F7
+    ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO,   # F8 - FF
+)
+
+# 0 : illegal
+# 1 : very unlikely
+# 2 : normal
+# 3 : very likely
+Latin1ClassModel = (
+    # UDF OTH ASC ASS ACV ACO ASV ASO
+    0,  0,  0,  0,  0,  0,  0,  0,  # UDF
+    0,  3,  3,  3,  3,  3,  3,  3,  # OTH
+    0,  3,  3,  3,  3,  3,  3,  3,  # ASC
+    0,  3,  3,  3,  1,  1,  3,  3,  # ASS
+    0,  3,  3,  3,  1,  2,  1,  2,  # ACV
+    0,  3,  3,  3,  3,  3,  3,  3,  # ACO
+    0,  3,  1,  3,  1,  1,  1,  3,  # ASV
+    0,  3,  1,  3,  1,  1,  3,  3,  # ASO
+)
+
+
+class Latin1Prober(CharSetProber):
+    def __init__(self):
+        CharSetProber.__init__(self)
+        self.reset()
+
+    def reset(self):
+        self._mLastCharClass = OTH
+        self._mFreqCounter = [0] * FREQ_CAT_NUM
+        CharSetProber.reset(self)
+
+    def get_charset_name(self):
+        return "windows-1252"
+
+    def feed(self, aBuf):
+        aBuf = self.filter_with_english_letters(aBuf)
+        for c in aBuf:
+            charClass = Latin1_CharToClass[wrap_ord(c)]
+            freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM)
+                                    + charClass]
+            if freq == 0:
+                self._mState = eNotMe
+                break
+            self._mFreqCounter[freq] += 1
+            self._mLastCharClass = charClass
+
+        return self.get_state()
+
+    def get_confidence(self):
+        if self.get_state() == eNotMe:
+            return 0.01
+
+        total = sum(self._mFreqCounter)
+        if total < 0.01:
+            confidence = 0.0
+        else:
+            confidence = ((self._mFreqCounter[3] - self._mFreqCounter[1] * 20.0)
+                          / total)
+        if confidence < 0.0:
+            confidence = 0.0
+        # lower the confidence of latin1 so that other more accurate
+        # detector can take priority.
+        confidence = confidence * 0.73
+        return confidence

chardet/mbcharsetprober.py

@@ -15,28 +15,29 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, sys
-from constants import eStart, eError, eItsMe
-from charsetprober import CharSetProber
+import sys
+from . import constants
+from .charsetprober import CharSetProber
+
 
 class MultiByteCharSetProber(CharSetProber):
     def __init__(self):
         CharSetProber.__init__(self)
         self._mDistributionAnalyzer = None
         self._mCodingSM = None
-        self._mLastChar = ['\x00', '\x00']
+        self._mLastChar = [0, 0]
 
     def reset(self):
         CharSetProber.reset(self)
@@ -44,7 +45,7 @@ class MultiByteCharSetProber(CharSetProber):
             self._mCodingSM.reset()
         if self._mDistributionAnalyzer:
             self._mDistributionAnalyzer.reset()
-        self._mLastChar = ['\x00', '\x00']
+        self._mLastChar = [0, 0]
 
     def get_charset_name(self):
         pass
@@ -53,27 +54,30 @@ class MultiByteCharSetProber(CharSetProber):
         aLen = len(aBuf)
         for i in range(0, aLen):
             codingState = self._mCodingSM.next_state(aBuf[i])
-            if codingState == eError:
+            if codingState == constants.eError:
                 if constants._debug:
-                    sys.stderr.write(self.get_charset_name() + ' prober hit error at byte ' + str(i) + '\n')
+                    sys.stderr.write(self.get_charset_name()
+                                     + ' prober hit error at byte ' + str(i)
+                                     + '\n')
                 self._mState = constants.eNotMe
                 break
-            elif codingState == eItsMe:
+            elif codingState == constants.eItsMe:
                 self._mState = constants.eFoundIt
                 break
-            elif codingState == eStart:
+            elif codingState == constants.eStart:
                 charLen = self._mCodingSM.get_current_charlen()
                 if i == 0:
                     self._mLastChar[1] = aBuf[0]
                     self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
                 else:
-                    self._mDistributionAnalyzer.feed(aBuf[i-1:i+1], charLen)
-                    
+                    self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
+                                                     charLen)
+
         self._mLastChar[0] = aBuf[aLen - 1]
-        
+
         if self.get_state() == constants.eDetecting:
-            if self._mDistributionAnalyzer.got_enough_data() and \
-               (self.get_confidence() > constants.SHORTCUT_THRESHOLD):
+            if (self._mDistributionAnalyzer.got_enough_data() and
+                    (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
                 self._mState = constants.eFoundIt
 
         return self.get_state()

chardet/mbcsgroupprober.py

@@ -15,36 +15,40 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-from charsetgroupprober import CharSetGroupProber
-from utf8prober import UTF8Prober
-from sjisprober import SJISProber
-from eucjpprober import EUCJPProber
-from gb2312prober import GB2312Prober
-from euckrprober import EUCKRProber
-from big5prober import Big5Prober
-from euctwprober import EUCTWProber
+from .charsetgroupprober import CharSetGroupProber
+from .utf8prober import UTF8Prober
+from .sjisprober import SJISProber
+from .eucjpprober import EUCJPProber
+from .gb2312prober import GB2312Prober
+from .euckrprober import EUCKRProber
+from .cp949prober import CP949Prober
+from .big5prober import Big5Prober
+from .euctwprober import EUCTWProber
+
 
 class MBCSGroupProber(CharSetGroupProber):
     def __init__(self):
         CharSetGroupProber.__init__(self)
-        self._mProbers = [ \
+        self._mProbers = [
             UTF8Prober(),
             SJISProber(),
             EUCJPProber(),
             GB2312Prober(),
             EUCKRProber(),
+            CP949Prober(),
             Big5Prober(),
-            EUCTWProber()]
+            EUCTWProber()
+        ]
         self.reset()

chardet/mbcssm.py

@@ -0,0 +1,572 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .constants import eStart, eError, eItsMe
+
+# BIG5
+
+BIG5_cls = (
+    1,1,1,1,1,1,1,1,  # 00 - 07    #allow 0x00 as legal value
+    1,1,1,1,1,1,0,0,  # 08 - 0f
+    1,1,1,1,1,1,1,1,  # 10 - 17
+    1,1,1,0,1,1,1,1,  # 18 - 1f
+    1,1,1,1,1,1,1,1,  # 20 - 27
+    1,1,1,1,1,1,1,1,  # 28 - 2f
+    1,1,1,1,1,1,1,1,  # 30 - 37
+    1,1,1,1,1,1,1,1,  # 38 - 3f
+    2,2,2,2,2,2,2,2,  # 40 - 47
+    2,2,2,2,2,2,2,2,  # 48 - 4f
+    2,2,2,2,2,2,2,2,  # 50 - 57
+    2,2,2,2,2,2,2,2,  # 58 - 5f
+    2,2,2,2,2,2,2,2,  # 60 - 67
+    2,2,2,2,2,2,2,2,  # 68 - 6f
+    2,2,2,2,2,2,2,2,  # 70 - 77
+    2,2,2,2,2,2,2,1,  # 78 - 7f
+    4,4,4,4,4,4,4,4,  # 80 - 87
+    4,4,4,4,4,4,4,4,  # 88 - 8f
+    4,4,4,4,4,4,4,4,  # 90 - 97
+    4,4,4,4,4,4,4,4,  # 98 - 9f
+    4,3,3,3,3,3,3,3,  # a0 - a7
+    3,3,3,3,3,3,3,3,  # a8 - af
+    3,3,3,3,3,3,3,3,  # b0 - b7
+    3,3,3,3,3,3,3,3,  # b8 - bf
+    3,3,3,3,3,3,3,3,  # c0 - c7
+    3,3,3,3,3,3,3,3,  # c8 - cf
+    3,3,3,3,3,3,3,3,  # d0 - d7
+    3,3,3,3,3,3,3,3,  # d8 - df
+    3,3,3,3,3,3,3,3,  # e0 - e7
+    3,3,3,3,3,3,3,3,  # e8 - ef
+    3,3,3,3,3,3,3,3,  # f0 - f7
+    3,3,3,3,3,3,3,0  # f8 - ff
+)
+
+BIG5_st = (
+    eError,eStart,eStart,     3,eError,eError,eError,eError,#00-07
+    eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,#08-0f
+    eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart#10-17
+)
+
+Big5CharLenTable = (0, 1, 1, 2, 0)
+
+Big5SMModel = {'classTable': BIG5_cls,
+               'classFactor': 5,
+               'stateTable': BIG5_st,
+               'charLenTable': Big5CharLenTable,
+               'name': 'Big5'}
+
+# CP949
+
+CP949_cls  = (
+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,0,0,  # 00 - 0f
+    1,1,1,1,1,1,1,1, 1,1,1,0,1,1,1,1,  # 10 - 1f
+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,  # 20 - 2f
+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,  # 30 - 3f
+    1,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,  # 40 - 4f
+    4,4,5,5,5,5,5,5, 5,5,5,1,1,1,1,1,  # 50 - 5f
+    1,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5,  # 60 - 6f
+    5,5,5,5,5,5,5,5, 5,5,5,1,1,1,1,1,  # 70 - 7f
+    0,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,  # 80 - 8f
+    6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,  # 90 - 9f
+    6,7,7,7,7,7,7,7, 7,7,7,7,7,8,8,8,  # a0 - af
+    7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,  # b0 - bf
+    7,7,7,7,7,7,9,2, 2,3,2,2,2,2,2,2,  # c0 - cf
+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,  # d0 - df
+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,  # e0 - ef
+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,0,  # f0 - ff
+)
+
+CP949_st = (
+#cls=    0      1      2      3      4      5      6      7      8      9  # previous state =
+    eError,eStart,     3,eError,eStart,eStart,     4,     5,eError,     6, # eStart
+    eError,eError,eError,eError,eError,eError,eError,eError,eError,eError, # eError
+    eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe, # eItsMe
+    eError,eError,eStart,eStart,eError,eError,eError,eStart,eStart,eStart, # 3
+    eError,eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart, # 4
+    eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart, # 5
+    eError,eStart,eStart,eStart,eStart,eError,eError,eStart,eStart,eStart, # 6
+)
+
+CP949CharLenTable = (0, 1, 2, 0, 1, 1, 2, 2, 0, 2)
+
+CP949SMModel = {'classTable': CP949_cls,
+                'classFactor': 10,
+                'stateTable': CP949_st,
+                'charLenTable': CP949CharLenTable,
+                'name': 'CP949'}
+
+# EUC-JP
+
+EUCJP_cls = (
+    4,4,4,4,4,4,4,4,  # 00 - 07
+    4,4,4,4,4,4,5,5,  # 08 - 0f
+    4,4,4,4,4,4,4,4,  # 10 - 17
+    4,4,4,5,4,4,4,4,  # 18 - 1f
+    4,4,4,4,4,4,4,4,  # 20 - 27
+    4,4,4,4,4,4,4,4,  # 28 - 2f
+    4,4,4,4,4,4,4,4,  # 30 - 37
+    4,4,4,4,4,4,4,4,  # 38 - 3f
+    4,4,4,4,4,4,4,4,  # 40 - 47
+    4,4,4,4,4,4,4,4,  # 48 - 4f
+    4,4,4,4,4,4,4,4,  # 50 - 57
+    4,4,4,4,4,4,4,4,  # 58 - 5f
+    4,4,4,4,4,4,4,4,  # 60 - 67
+    4,4,4,4,4,4,4,4,  # 68 - 6f
+    4,4,4,4,4,4,4,4,  # 70 - 77
+    4,4,4,4,4,4,4,4,  # 78 - 7f
+    5,5,5,5,5,5,5,5,  # 80 - 87
+    5,5,5,5,5,5,1,3,  # 88 - 8f
+    5,5,5,5,5,5,5,5,  # 90 - 97
+    5,5,5,5,5,5,5,5,  # 98 - 9f
+    5,2,2,2,2,2,2,2,  # a0 - a7
+    2,2,2,2,2,2,2,2,  # a8 - af
+    2,2,2,2,2,2,2,2,  # b0 - b7
+    2,2,2,2,2,2,2,2,  # b8 - bf
+    2,2,2,2,2,2,2,2,  # c0 - c7
+    2,2,2,2,2,2,2,2,  # c8 - cf
+    2,2,2,2,2,2,2,2,  # d0 - d7
+    2,2,2,2,2,2,2,2,  # d8 - df
+    0,0,0,0,0,0,0,0,  # e0 - e7
+    0,0,0,0,0,0,0,0,  # e8 - ef
+    0,0,0,0,0,0,0,0,  # f0 - f7
+    0,0,0,0,0,0,0,5  # f8 - ff
+)
+
+EUCJP_st = (
+          3,     4,     3,     5,eStart,eError,eError,eError,#00-07
+     eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f
+     eItsMe,eItsMe,eStart,eError,eStart,eError,eError,eError,#10-17
+     eError,eError,eStart,eError,eError,eError,     3,eError,#18-1f
+          3,eError,eError,eError,eStart,eStart,eStart,eStart#20-27
+)
+
+EUCJPCharLenTable = (2, 2, 2, 3, 1, 0)
+
+EUCJPSMModel = {'classTable': EUCJP_cls,
+                'classFactor': 6,
+                'stateTable': EUCJP_st,
+                'charLenTable': EUCJPCharLenTable,
+                'name': 'EUC-JP'}
+
+# EUC-KR
+
+EUCKR_cls  = (
+    1,1,1,1,1,1,1,1,  # 00 - 07
+    1,1,1,1,1,1,0,0,  # 08 - 0f
+    1,1,1,1,1,1,1,1,  # 10 - 17
+    1,1,1,0,1,1,1,1,  # 18 - 1f
+    1,1,1,1,1,1,1,1,  # 20 - 27
+    1,1,1,1,1,1,1,1,  # 28 - 2f
+    1,1,1,1,1,1,1,1,  # 30 - 37
+    1,1,1,1,1,1,1,1,  # 38 - 3f
+    1,1,1,1,1,1,1,1,  # 40 - 47
+    1,1,1,1,1,1,1,1,  # 48 - 4f
+    1,1,1,1,1,1,1,1,  # 50 - 57
+    1,1,1,1,1,1,1,1,  # 58 - 5f
+    1,1,1,1,1,1,1,1,  # 60 - 67
+    1,1,1,1,1,1,1,1,  # 68 - 6f
+    1,1,1,1,1,1,1,1,  # 70 - 77
+    1,1,1,1,1,1,1,1,  # 78 - 7f
+    0,0,0,0,0,0,0,0,  # 80 - 87
+    0,0,0,0,0,0,0,0,  # 88 - 8f
+    0,0,0,0,0,0,0,0,  # 90 - 97
+    0,0,0,0,0,0,0,0,  # 98 - 9f
+    0,2,2,2,2,2,2,2,  # a0 - a7
+    2,2,2,2,2,3,3,3,  # a8 - af
+    2,2,2,2,2,2,2,2,  # b0 - b7
+    2,2,2,2,2,2,2,2,  # b8 - bf
+    2,2,2,2,2,2,2,2,  # c0 - c7
+    2,3,2,2,2,2,2,2,  # c8 - cf
+    2,2,2,2,2,2,2,2,  # d0 - d7
+    2,2,2,2,2,2,2,2,  # d8 - df
+    2,2,2,2,2,2,2,2,  # e0 - e7
+    2,2,2,2,2,2,2,2,  # e8 - ef
+    2,2,2,2,2,2,2,2,  # f0 - f7
+    2,2,2,2,2,2,2,0   # f8 - ff
+)
+
+EUCKR_st = (
+    eError,eStart,     3,eError,eError,eError,eError,eError,#00-07
+    eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,eStart #08-0f
+)
+
+EUCKRCharLenTable = (0, 1, 2, 0)
+
+EUCKRSMModel = {'classTable': EUCKR_cls,
+                'classFactor': 4,
+                'stateTable': EUCKR_st,
+                'charLenTable': EUCKRCharLenTable,
+                'name': 'EUC-KR'}
+
+# EUC-TW
+
+EUCTW_cls = (
+    2,2,2,2,2,2,2,2,  # 00 - 07
+    2,2,2,2,2,2,0,0,  # 08 - 0f
+    2,2,2,2,2,2,2,2,  # 10 - 17
+    2,2,2,0,2,2,2,2,  # 18 - 1f
+    2,2,2,2,2,2,2,2,  # 20 - 27
+    2,2,2,2,2,2,2,2,  # 28 - 2f
+    2,2,2,2,2,2,2,2,  # 30 - 37
+    2,2,2,2,2,2,2,2,  # 38 - 3f
+    2,2,2,2,2,2,2,2,  # 40 - 47
+    2,2,2,2,2,2,2,2,  # 48 - 4f
+    2,2,2,2,2,2,2,2,  # 50 - 57
+    2,2,2,2,2,2,2,2,  # 58 - 5f
+    2,2,2,2,2,2,2,2,  # 60 - 67
+    2,2,2,2,2,2,2,2,  # 68 - 6f
+    2,2,2,2,2,2,2,2,  # 70 - 77
+    2,2,2,2,2,2,2,2,  # 78 - 7f
+    0,0,0,0,0,0,0,0,  # 80 - 87
+    0,0,0,0,0,0,6,0,  # 88 - 8f
+    0,0,0,0,0,0,0,0,  # 90 - 97
+    0,0,0,0,0,0,0,0,  # 98 - 9f
+    0,3,4,4,4,4,4,4,  # a0 - a7
+    5,5,1,1,1,1,1,1,  # a8 - af
+    1,1,1,1,1,1,1,1,  # b0 - b7
+    1,1,1,1,1,1,1,1,  # b8 - bf
+    1,1,3,1,3,3,3,3,  # c0 - c7
+    3,3,3,3,3,3,3,3,  # c8 - cf
+    3,3,3,3,3,3,3,3,  # d0 - d7
+    3,3,3,3,3,3,3,3,  # d8 - df
+    3,3,3,3,3,3,3,3,  # e0 - e7
+    3,3,3,3,3,3,3,3,  # e8 - ef
+    3,3,3,3,3,3,3,3,  # f0 - f7
+    3,3,3,3,3,3,3,0   # f8 - ff
+)
+
+EUCTW_st = (
+    eError,eError,eStart,     3,     3,     3,     4,eError,#00-07
+    eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,#08-0f
+    eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eStart,eError,#10-17
+    eStart,eStart,eStart,eError,eError,eError,eError,eError,#18-1f
+         5,eError,eError,eError,eStart,eError,eStart,eStart,#20-27
+    eStart,eError,eStart,eStart,eStart,eStart,eStart,eStart #28-2f
+)
+
+EUCTWCharLenTable = (0, 0, 1, 2, 2, 2, 3)
+
+EUCTWSMModel = {'classTable': EUCTW_cls,
+                'classFactor': 7,
+                'stateTable': EUCTW_st,
+                'charLenTable': EUCTWCharLenTable,
+                'name': 'x-euc-tw'}
+
+# GB2312
+
+GB2312_cls = (
+    1,1,1,1,1,1,1,1,  # 00 - 07
+    1,1,1,1,1,1,0,0,  # 08 - 0f
+    1,1,1,1,1,1,1,1,  # 10 - 17
+    1,1,1,0,1,1,1,1,  # 18 - 1f
+    1,1,1,1,1,1,1,1,  # 20 - 27
+    1,1,1,1,1,1,1,1,  # 28 - 2f
+    3,3,3,3,3,3,3,3,  # 30 - 37
+    3,3,1,1,1,1,1,1,  # 38 - 3f
+    2,2,2,2,2,2,2,2,  # 40 - 47
+    2,2,2,2,2,2,2,2,  # 48 - 4f
+    2,2,2,2,2,2,2,2,  # 50 - 57
+    2,2,2,2,2,2,2,2,  # 58 - 5f
+    2,2,2,2,2,2,2,2,  # 60 - 67
+    2,2,2,2,2,2,2,2,  # 68 - 6f
+    2,2,2,2,2,2,2,2,  # 70 - 77
+    2,2,2,2,2,2,2,4,  # 78 - 7f
+    5,6,6,6,6,6,6,6,  # 80 - 87
+    6,6,6,6,6,6,6,6,  # 88 - 8f
+    6,6,6,6,6,6,6,6,  # 90 - 97
+    6,6,6,6,6,6,6,6,  # 98 - 9f
+    6,6,6,6,6,6,6,6,  # a0 - a7
+    6,6,6,6,6,6,6,6,  # a8 - af
+    6,6,6,6,6,6,6,6,  # b0 - b7
+    6,6,6,6,6,6,6,6,  # b8 - bf
+    6,6,6,6,6,6,6,6,  # c0 - c7
+    6,6,6,6,6,6,6,6,  # c8 - cf
+    6,6,6,6,6,6,6,6,  # d0 - d7
+    6,6,6,6,6,6,6,6,  # d8 - df
+    6,6,6,6,6,6,6,6,  # e0 - e7
+    6,6,6,6,6,6,6,6,  # e8 - ef
+    6,6,6,6,6,6,6,6,  # f0 - f7
+    6,6,6,6,6,6,6,0   # f8 - ff
+)
+
+GB2312_st = (
+    eError,eStart,eStart,eStart,eStart,eStart,     3,eError,#00-07
+    eError,eError,eError,eError,eError,eError,eItsMe,eItsMe,#08-0f
+    eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eError,eError,eStart,#10-17
+         4,eError,eStart,eStart,eError,eError,eError,eError,#18-1f
+    eError,eError,     5,eError,eError,eError,eItsMe,eError,#20-27
+    eError,eError,eStart,eStart,eStart,eStart,eStart,eStart #28-2f
+)
+
+# To be accurate, the length of class 6 can be either 2 or 4.
+# But it is not necessary to discriminate between the two since
+# it is used for frequency analysis only, and we are validing
+# each code range there as well. So it is safe to set it to be
+# 2 here.
+GB2312CharLenTable = (0, 1, 1, 1, 1, 1, 2)
+
+GB2312SMModel = {'classTable': GB2312_cls,
+                  'classFactor': 7,
+                  'stateTable': GB2312_st,
+                  'charLenTable': GB2312CharLenTable,
+                  'name': 'GB2312'}
+
+# Shift_JIS
+
+SJIS_cls = (
+    1,1,1,1,1,1,1,1,  # 00 - 07
+    1,1,1,1,1,1,0,0,  # 08 - 0f
+    1,1,1,1,1,1,1,1,  # 10 - 17
+    1,1,1,0,1,1,1,1,  # 18 - 1f
+    1,1,1,1,1,1,1,1,  # 20 - 27
+    1,1,1,1,1,1,1,1,  # 28 - 2f
+    1,1,1,1,1,1,1,1,  # 30 - 37
+    1,1,1,1,1,1,1,1,  # 38 - 3f
+    2,2,2,2,2,2,2,2,  # 40 - 47
+    2,2,2,2,2,2,2,2,  # 48 - 4f
+    2,2,2,2,2,2,2,2,  # 50 - 57
+    2,2,2,2,2,2,2,2,  # 58 - 5f
+    2,2,2,2,2,2,2,2,  # 60 - 67
+    2,2,2,2,2,2,2,2,  # 68 - 6f
+    2,2,2,2,2,2,2,2,  # 70 - 77
+    2,2,2,2,2,2,2,1,  # 78 - 7f
+    3,3,3,3,3,2,2,3,  # 80 - 87
+    3,3,3,3,3,3,3,3,  # 88 - 8f
+    3,3,3,3,3,3,3,3,  # 90 - 97
+    3,3,3,3,3,3,3,3,  # 98 - 9f
+    #0xa0 is illegal in sjis encoding, but some pages does
+    #contain such byte. We need to be more error forgiven.
+    2,2,2,2,2,2,2,2,  # a0 - a7
+    2,2,2,2,2,2,2,2,  # a8 - af
+    2,2,2,2,2,2,2,2,  # b0 - b7
+    2,2,2,2,2,2,2,2,  # b8 - bf
+    2,2,2,2,2,2,2,2,  # c0 - c7
+    2,2,2,2,2,2,2,2,  # c8 - cf
+    2,2,2,2,2,2,2,2,  # d0 - d7
+    2,2,2,2,2,2,2,2,  # d8 - df
+    3,3,3,3,3,3,3,3,  # e0 - e7
+    3,3,3,3,3,4,4,4,  # e8 - ef
+    3,3,3,3,3,3,3,3,  # f0 - f7
+    3,3,3,3,3,0,0,0)  # f8 - ff
+
+
+SJIS_st = (
+    eError,eStart,eStart,     3,eError,eError,eError,eError,#00-07
+    eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f
+    eItsMe,eItsMe,eError,eError,eStart,eStart,eStart,eStart #10-17
+)
+
+SJISCharLenTable = (0, 1, 1, 2, 0, 0)
+
+SJISSMModel = {'classTable': SJIS_cls,
+               'classFactor': 6,
+               'stateTable': SJIS_st,
+               'charLenTable': SJISCharLenTable,
+               'name': 'Shift_JIS'}
+
+# UCS2-BE
+
+UCS2BE_cls = (
+    0,0,0,0,0,0,0,0,  # 00 - 07
+    0,0,1,0,0,2,0,0,  # 08 - 0f
+    0,0,0,0,0,0,0,0,  # 10 - 17
+    0,0,0,3,0,0,0,0,  # 18 - 1f
+    0,0,0,0,0,0,0,0,  # 20 - 27
+    0,3,3,3,3,3,0,0,  # 28 - 2f
+    0,0,0,0,0,0,0,0,  # 30 - 37
+    0,0,0,0,0,0,0,0,  # 38 - 3f
+    0,0,0,0,0,0,0,0,  # 40 - 47
+    0,0,0,0,0,0,0,0,  # 48 - 4f
+    0,0,0,0,0,0,0,0,  # 50 - 57
+    0,0,0,0,0,0,0,0,  # 58 - 5f
+    0,0,0,0,0,0,0,0,  # 60 - 67
+    0,0,0,0,0,0,0,0,  # 68 - 6f
+    0,0,0,0,0,0,0,0,  # 70 - 77
+    0,0,0,0,0,0,0,0,  # 78 - 7f
+    0,0,0,0,0,0,0,0,  # 80 - 87
+    0,0,0,0,0,0,0,0,  # 88 - 8f
+    0,0,0,0,0,0,0,0,  # 90 - 97
+    0,0,0,0,0,0,0,0,  # 98 - 9f
+    0,0,0,0,0,0,0,0,  # a0 - a7
+    0,0,0,0,0,0,0,0,  # a8 - af
+    0,0,0,0,0,0,0,0,  # b0 - b7
+    0,0,0,0,0,0,0,0,  # b8 - bf
+    0,0,0,0,0,0,0,0,  # c0 - c7
+    0,0,0,0,0,0,0,0,  # c8 - cf
+    0,0,0,0,0,0,0,0,  # d0 - d7
+    0,0,0,0,0,0,0,0,  # d8 - df
+    0,0,0,0,0,0,0,0,  # e0 - e7
+    0,0,0,0,0,0,0,0,  # e8 - ef
+    0,0,0,0,0,0,0,0,  # f0 - f7
+    0,0,0,0,0,0,4,5   # f8 - ff
+)
+
+UCS2BE_st  = (
+          5,     7,     7,eError,     4,     3,eError,eError,#00-07
+     eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f
+     eItsMe,eItsMe,     6,     6,     6,     6,eError,eError,#10-17
+          6,     6,     6,     6,     6,eItsMe,     6,     6,#18-1f
+          6,     6,     6,     6,     5,     7,     7,eError,#20-27
+          5,     8,     6,     6,eError,     6,     6,     6,#28-2f
+          6,     6,     6,     6,eError,eError,eStart,eStart #30-37
+)
+
+UCS2BECharLenTable = (2, 2, 2, 0, 2, 2)
+
+UCS2BESMModel = {'classTable': UCS2BE_cls,
+                 'classFactor': 6,
+                 'stateTable': UCS2BE_st,
+                 'charLenTable': UCS2BECharLenTable,
+                 'name': 'UTF-16BE'}
+
+# UCS2-LE
+
+UCS2LE_cls = (
+    0,0,0,0,0,0,0,0,  # 00 - 07
+    0,0,1,0,0,2,0,0,  # 08 - 0f
+    0,0,0,0,0,0,0,0,  # 10 - 17
+    0,0,0,3,0,0,0,0,  # 18 - 1f
+    0,0,0,0,0,0,0,0,  # 20 - 27
+    0,3,3,3,3,3,0,0,  # 28 - 2f
+    0,0,0,0,0,0,0,0,  # 30 - 37
+    0,0,0,0,0,0,0,0,  # 38 - 3f
+    0,0,0,0,0,0,0,0,  # 40 - 47
+    0,0,0,0,0,0,0,0,  # 48 - 4f
+    0,0,0,0,0,0,0,0,  # 50 - 57
+    0,0,0,0,0,0,0,0,  # 58 - 5f
+    0,0,0,0,0,0,0,0,  # 60 - 67
+    0,0,0,0,0,0,0,0,  # 68 - 6f
+    0,0,0,0,0,0,0,0,  # 70 - 77
+    0,0,0,0,0,0,0,0,  # 78 - 7f
+    0,0,0,0,0,0,0,0,  # 80 - 87
+    0,0,0,0,0,0,0,0,  # 88 - 8f
+    0,0,0,0,0,0,0,0,  # 90 - 97
+    0,0,0,0,0,0,0,0,  # 98 - 9f
+    0,0,0,0,0,0,0,0,  # a0 - a7
+    0,0,0,0,0,0,0,0,  # a8 - af
+    0,0,0,0,0,0,0,0,  # b0 - b7
+    0,0,0,0,0,0,0,0,  # b8 - bf
+    0,0,0,0,0,0,0,0,  # c0 - c7
+    0,0,0,0,0,0,0,0,  # c8 - cf
+    0,0,0,0,0,0,0,0,  # d0 - d7
+    0,0,0,0,0,0,0,0,  # d8 - df
+    0,0,0,0,0,0,0,0,  # e0 - e7
+    0,0,0,0,0,0,0,0,  # e8 - ef
+    0,0,0,0,0,0,0,0,  # f0 - f7
+    0,0,0,0,0,0,4,5   # f8 - ff
+)
+
+UCS2LE_st = (
+          6,     6,     7,     6,     4,     3,eError,eError,#00-07
+     eError,eError,eError,eError,eItsMe,eItsMe,eItsMe,eItsMe,#08-0f
+     eItsMe,eItsMe,     5,     5,     5,eError,eItsMe,eError,#10-17
+          5,     5,     5,eError,     5,eError,     6,     6,#18-1f
+          7,     6,     8,     8,     5,     5,     5,eError,#20-27
+          5,     5,     5,eError,eError,eError,     5,     5,#28-2f
+          5,     5,     5,eError,     5,eError,eStart,eStart #30-37
+)
+
+UCS2LECharLenTable = (2, 2, 2, 2, 2, 2)
+
+UCS2LESMModel = {'classTable': UCS2LE_cls,
+                 'classFactor': 6,
+                 'stateTable': UCS2LE_st,
+                 'charLenTable': UCS2LECharLenTable,
+                 'name': 'UTF-16LE'}
+
+# UTF-8
+
+UTF8_cls = (
+    1,1,1,1,1,1,1,1,  # 00 - 07  #allow 0x00 as a legal value
+    1,1,1,1,1,1,0,0,  # 08 - 0f
+    1,1,1,1,1,1,1,1,  # 10 - 17
+    1,1,1,0,1,1,1,1,  # 18 - 1f
+    1,1,1,1,1,1,1,1,  # 20 - 27
+    1,1,1,1,1,1,1,1,  # 28 - 2f
+    1,1,1,1,1,1,1,1,  # 30 - 37
+    1,1,1,1,1,1,1,1,  # 38 - 3f
+    1,1,1,1,1,1,1,1,  # 40 - 47
+    1,1,1,1,1,1,1,1,  # 48 - 4f
+    1,1,1,1,1,1,1,1,  # 50 - 57
+    1,1,1,1,1,1,1,1,  # 58 - 5f
+    1,1,1,1,1,1,1,1,  # 60 - 67
+    1,1,1,1,1,1,1,1,  # 68 - 6f
+    1,1,1,1,1,1,1,1,  # 70 - 77
+    1,1,1,1,1,1,1,1,  # 78 - 7f
+    2,2,2,2,3,3,3,3,  # 80 - 87
+    4,4,4,4,4,4,4,4,  # 88 - 8f
+    4,4,4,4,4,4,4,4,  # 90 - 97
+    4,4,4,4,4,4,4,4,  # 98 - 9f
+    5,5,5,5,5,5,5,5,  # a0 - a7
+    5,5,5,5,5,5,5,5,  # a8 - af
+    5,5,5,5,5,5,5,5,  # b0 - b7
+    5,5,5,5,5,5,5,5,  # b8 - bf
+    0,0,6,6,6,6,6,6,  # c0 - c7
+    6,6,6,6,6,6,6,6,  # c8 - cf
+    6,6,6,6,6,6,6,6,  # d0 - d7
+    6,6,6,6,6,6,6,6,  # d8 - df
+    7,8,8,8,8,8,8,8,  # e0 - e7
+    8,8,8,8,8,9,8,8,  # e8 - ef
+    10,11,11,11,11,11,11,11,  # f0 - f7
+    12,13,13,13,14,15,0,0    # f8 - ff
+)
+
+UTF8_st = (
+    eError,eStart,eError,eError,eError,eError,     12,   10,#00-07
+         9,     11,     8,     7,     6,     5,     4,    3,#08-0f
+    eError,eError,eError,eError,eError,eError,eError,eError,#10-17
+    eError,eError,eError,eError,eError,eError,eError,eError,#18-1f
+    eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,#20-27
+    eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,#28-2f
+    eError,eError,     5,     5,     5,     5,eError,eError,#30-37
+    eError,eError,eError,eError,eError,eError,eError,eError,#38-3f
+    eError,eError,eError,     5,     5,     5,eError,eError,#40-47
+    eError,eError,eError,eError,eError,eError,eError,eError,#48-4f
+    eError,eError,     7,     7,     7,     7,eError,eError,#50-57
+    eError,eError,eError,eError,eError,eError,eError,eError,#58-5f
+    eError,eError,eError,eError,     7,     7,eError,eError,#60-67
+    eError,eError,eError,eError,eError,eError,eError,eError,#68-6f
+    eError,eError,     9,     9,     9,     9,eError,eError,#70-77
+    eError,eError,eError,eError,eError,eError,eError,eError,#78-7f
+    eError,eError,eError,eError,eError,     9,eError,eError,#80-87
+    eError,eError,eError,eError,eError,eError,eError,eError,#88-8f
+    eError,eError,    12,    12,    12,    12,eError,eError,#90-97
+    eError,eError,eError,eError,eError,eError,eError,eError,#98-9f
+    eError,eError,eError,eError,eError,    12,eError,eError,#a0-a7
+    eError,eError,eError,eError,eError,eError,eError,eError,#a8-af
+    eError,eError,    12,    12,    12,eError,eError,eError,#b0-b7
+    eError,eError,eError,eError,eError,eError,eError,eError,#b8-bf
+    eError,eError,eStart,eStart,eStart,eStart,eError,eError,#c0-c7
+    eError,eError,eError,eError,eError,eError,eError,eError #c8-cf
+)
+
+UTF8CharLenTable = (0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6)
+
+UTF8SMModel = {'classTable': UTF8_cls,
+               'classFactor': 16,
+               'stateTable': UTF8_st,
+               'charLenTable': UTF8CharLenTable,
+               'name': 'UTF-8'}

chardet/sbcharsetprober.py

@@ -14,20 +14,22 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, sys
-from charsetprober import CharSetProber
+import sys
+from . import constants
+from .charsetprober import CharSetProber
+from .compat import wrap_ord
 
 SAMPLE_SIZE = 64
 SB_ENOUGH_REL_THRESHOLD = 1024
@@ -37,22 +39,27 @@ SYMBOL_CAT_ORDER = 250
 NUMBER_OF_SEQ_CAT = 4
 POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1
 #NEGATIVE_CAT = 0
- 
+
+
 class SingleByteCharSetProber(CharSetProber):
-    def __init__(self, model, reversed=constants.False, nameProber=None):
+    def __init__(self, model, reversed=False, nameProber=None):
         CharSetProber.__init__(self)
         self._mModel = model
-        self._mReversed = reversed # TRUE if we need to reverse every pair in the model lookup
-        self._mNameProber = nameProber # Optional auxiliary prober for name decision
+        # TRUE if we need to reverse every pair in the model lookup
+        self._mReversed = reversed
+        # Optional auxiliary prober for name decision
+        self._mNameProber = nameProber
         self.reset()
 
     def reset(self):
         CharSetProber.reset(self)
-        self._mLastOrder = 255 # char order of last character
+        # char order of last character
+        self._mLastOrder = 255
         self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT
         self._mTotalSeqs = 0
         self._mTotalChar = 0
-        self._mFreqChar = 0 # characters that fall in our sampling range
+        # characters that fall in our sampling range
+        self._mFreqChar = 0
 
     def get_charset_name(self):
         if self._mNameProber:
@@ -67,7 +74,7 @@ class SingleByteCharSetProber(CharSetProber):
         if not aLen:
             return self.get_state()
         for c in aBuf:
-            order = self._mModel['charToOrderMap'][ord(c)]
+            order = self._mModel['charToOrderMap'][wrap_ord(c)]
             if order < SYMBOL_CAT_ORDER:
                 self._mTotalChar += 1
             if order < SAMPLE_SIZE:
@@ -75,9 +82,12 @@ class SingleByteCharSetProber(CharSetProber):
                 if self._mLastOrder < SAMPLE_SIZE:
                     self._mTotalSeqs += 1
                     if not self._mReversed:
-                        self._mSeqCounters[self._mModel['precedenceMatrix'][(self._mLastOrder * SAMPLE_SIZE) + order]] += 1
-                    else: # reverse the order of the letters in the lookup
-                        self._mSeqCounters[self._mModel['precedenceMatrix'][(order * SAMPLE_SIZE) + self._mLastOrder]] += 1
+                        i = (self._mLastOrder * SAMPLE_SIZE) + order
+                        model = self._mModel['precedenceMatrix'][i]
+                    else:  # reverse the order of the letters in the lookup
+                        i = (order * SAMPLE_SIZE) + self._mLastOrder
+                        model = self._mModel['precedenceMatrix'][i]
+                    self._mSeqCounters[model] += 1
             self._mLastOrder = order
 
         if self.get_state() == constants.eDetecting:
@@ -85,11 +95,16 @@ class SingleByteCharSetProber(CharSetProber):
                 cf = self.get_confidence()
                 if cf > POSITIVE_SHORTCUT_THRESHOLD:
                     if constants._debug:
-                        sys.stderr.write('%s confidence = %s, we have a winner\n' % (self._mModel['charsetName'], cf))
+                        sys.stderr.write('%s confidence = %s, we have a'
+                                         'winner\n' %
+                                         (self._mModel['charsetName'], cf))
                     self._mState = constants.eFoundIt
                 elif cf < NEGATIVE_SHORTCUT_THRESHOLD:
                     if constants._debug:
-                        sys.stderr.write('%s confidence = %s, below negative shortcut threshhold %s\n' % (self._mModel['charsetName'], cf, NEGATIVE_SHORTCUT_THRESHOLD))
+                        sys.stderr.write('%s confidence = %s, below negative'
+                                         'shortcut threshhold %s\n' %
+                                         (self._mModel['charsetName'], cf,
+                                          NEGATIVE_SHORTCUT_THRESHOLD))
                     self._mState = constants.eNotMe
 
         return self.get_state()
@@ -97,9 +112,8 @@ class SingleByteCharSetProber(CharSetProber):
     def get_confidence(self):
         r = 0.01
         if self._mTotalSeqs > 0:
-#            print self._mSeqCounters[POSITIVE_CAT], self._mTotalSeqs, self._mModel['mTypicalPositiveRatio']
-            r = (1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs / self._mModel['mTypicalPositiveRatio']
-#            print r, self._mFreqChar, self._mTotalChar
+            r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs
+                 / self._mModel['mTypicalPositiveRatio'])
             r = r * self._mFreqChar / self._mTotalChar
             if r >= 1.0:
                 r = 0.99

chardet/sbcsgroupprober.py

@@ -14,33 +14,35 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, sys
-from charsetgroupprober import CharSetGroupProber
-from sbcharsetprober import SingleByteCharSetProber
-from langcyrillicmodel import Win1251CyrillicModel, Koi8rModel, Latin5CyrillicModel, MacCyrillicModel, Ibm866Model, Ibm855Model
-from langgreekmodel import Latin7GreekModel, Win1253GreekModel
-from langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel
-from langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel
-from langthaimodel import TIS620ThaiModel
-from langhebrewmodel import Win1255HebrewModel
-from hebrewprober import HebrewProber
+from .charsetgroupprober import CharSetGroupProber
+from .sbcharsetprober import SingleByteCharSetProber
+from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel,
+                                Latin5CyrillicModel, MacCyrillicModel,
+                                Ibm866Model, Ibm855Model)
+from .langgreekmodel import Latin7GreekModel, Win1253GreekModel
+from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel
+from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel
+from .langthaimodel import TIS620ThaiModel
+from .langhebrewmodel import Win1255HebrewModel
+from .hebrewprober import HebrewProber
+
 
 class SBCSGroupProber(CharSetGroupProber):
     def __init__(self):
         CharSetGroupProber.__init__(self)
-        self._mProbers = [ \
+        self._mProbers = [
             SingleByteCharSetProber(Win1251CyrillicModel),
             SingleByteCharSetProber(Koi8rModel),
             SingleByteCharSetProber(Latin5CyrillicModel),
@@ -54,11 +56,14 @@ class SBCSGroupProber(CharSetGroupProber):
             SingleByteCharSetProber(Latin2HungarianModel),
             SingleByteCharSetProber(Win1250HungarianModel),
             SingleByteCharSetProber(TIS620ThaiModel),
-            ]
+        ]
         hebrewProber = HebrewProber()
-        logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, constants.False, hebrewProber)
-        visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, constants.True, hebrewProber)
+        logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel,
+                                                      False, hebrewProber)
+        visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, True,
+                                                     hebrewProber)
         hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber)
-        self._mProbers.extend([hebrewProber, logicalHebrewProber, visualHebrewProber])
+        self._mProbers.extend([hebrewProber, logicalHebrewProber,
+                               visualHebrewProber])
 
         self.reset()

chardet/sjisprober.py

@@ -13,25 +13,26 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-from mbcharsetprober import MultiByteCharSetProber
-from codingstatemachine import CodingStateMachine
-from chardistribution import SJISDistributionAnalysis
-from jpcntx import SJISContextAnalysis
-from mbcssm import SJISSMModel
-import constants, sys
-from constants import eStart, eError, eItsMe
+import sys
+from .mbcharsetprober import MultiByteCharSetProber
+from .codingstatemachine import CodingStateMachine
+from .chardistribution import SJISDistributionAnalysis
+from .jpcntx import SJISContextAnalysis
+from .mbcssm import SJISSMModel
+from . import constants
+
 
 class SJISProber(MultiByteCharSetProber):
     def __init__(self):
@@ -44,37 +45,42 @@ class SJISProber(MultiByteCharSetProber):
     def reset(self):
         MultiByteCharSetProber.reset(self)
         self._mContextAnalyzer.reset()
-        
+
     def get_charset_name(self):
-        return "SHIFT_JIS"
+        return self._mContextAnalyzer.get_charset_name()
 
     def feed(self, aBuf):
         aLen = len(aBuf)
         for i in range(0, aLen):
             codingState = self._mCodingSM.next_state(aBuf[i])
-            if codingState == eError:
+            if codingState == constants.eError:
                 if constants._debug:
-                    sys.stderr.write(self.get_charset_name() + ' prober hit error at byte ' + str(i) + '\n')
+                    sys.stderr.write(self.get_charset_name()
+                                     + ' prober hit error at byte ' + str(i)
+                                     + '\n')
                 self._mState = constants.eNotMe
                 break
-            elif codingState == eItsMe:
+            elif codingState == constants.eItsMe:
                 self._mState = constants.eFoundIt
                 break
-            elif codingState == eStart:
+            elif codingState == constants.eStart:
                 charLen = self._mCodingSM.get_current_charlen()
                 if i == 0:
                     self._mLastChar[1] = aBuf[0]
-                    self._mContextAnalyzer.feed(self._mLastChar[2 - charLen :], charLen)
+                    self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:],
+                                                charLen)
                     self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
                 else:
-                    self._mContextAnalyzer.feed(aBuf[i + 1 - charLen : i + 3 - charLen], charLen)
-                    self._mDistributionAnalyzer.feed(aBuf[i - 1 : i + 1], charLen)
-                    
+                    self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3
+                                                     - charLen], charLen)
+                    self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
+                                                     charLen)
+
         self._mLastChar[0] = aBuf[aLen - 1]
-        
+
         if self.get_state() == constants.eDetecting:
-            if self._mContextAnalyzer.got_enough_data() and \
-                   (self.get_confidence() > constants.SHORTCUT_THRESHOLD):
+            if (self._mContextAnalyzer.got_enough_data() and
+               (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
                 self._mState = constants.eFoundIt
 
         return self.get_state()

chardet/universaldetector.py

@@ -14,23 +14,25 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, sys
-from latin1prober import Latin1Prober # windows-1252
-from mbcsgroupprober import MBCSGroupProber # multi-byte character sets
-from sbcsgroupprober import SBCSGroupProber # single-byte character sets
-from escprober import EscCharSetProber # ISO-2122, etc.
+from . import constants
+import sys
+import codecs
+from .latin1prober import Latin1Prober  # windows-1252
+from .mbcsgroupprober import MBCSGroupProber  # multi-byte character sets
+from .sbcsgroupprober import SBCSGroupProber  # single-byte character sets
+from .escprober import EscCharSetProber  # ISO-2122, etc.
 import re
 
 MINIMUM_THRESHOLD = 0.20
@@ -38,68 +40,78 @@ ePureAscii = 0
 eEscAscii = 1
 eHighbyte = 2
 
+
 class UniversalDetector:
     def __init__(self):
-        self._highBitDetector = re.compile(r'[\x80-\xFF]')
-        self._escDetector = re.compile(r'(\033|~{)')
+        self._highBitDetector = re.compile(b'[\x80-\xFF]')
+        self._escDetector = re.compile(b'(\033|~{)')
         self._mEscCharSetProber = None
         self._mCharSetProbers = []
         self.reset()
 
     def reset(self):
         self.result = {'encoding': None, 'confidence': 0.0}
-        self.done = constants.False
-        self._mStart = constants.True
-        self._mGotData = constants.False
+        self.done = False
+        self._mStart = True
+        self._mGotData = False
         self._mInputState = ePureAscii
-        self._mLastChar = ''
+        self._mLastChar = b''
         if self._mEscCharSetProber:
             self._mEscCharSetProber.reset()
         for prober in self._mCharSetProbers:
             prober.reset()
 
     def feed(self, aBuf):
-        if self.done: return
+        if self.done:
+            return
 
         aLen = len(aBuf)
-        if not aLen: return
-        
+        if not aLen:
+            return
+
         if not self._mGotData:
             # If the data starts with BOM, we know it is UTF
-            if aBuf[:3] == '\xEF\xBB\xBF':
+            if aBuf[:3] == codecs.BOM_UTF8:
                 # EF BB BF  UTF-8 with BOM
-                self.result = {'encoding': "UTF-8", 'confidence': 1.0}
-            elif aBuf[:4] == '\xFF\xFE\x00\x00':
+                self.result = {'encoding': "UTF-8-SIG", 'confidence': 1.0}
+            elif aBuf[:4] == codecs.BOM_UTF32_LE:
                 # FF FE 00 00  UTF-32, little-endian BOM
                 self.result = {'encoding': "UTF-32LE", 'confidence': 1.0}
-            elif aBuf[:4] == '\x00\x00\xFE\xFF': 
+            elif aBuf[:4] == codecs.BOM_UTF32_BE:
                 # 00 00 FE FF  UTF-32, big-endian BOM
                 self.result = {'encoding': "UTF-32BE", 'confidence': 1.0}
-            elif aBuf[:4] == '\xFE\xFF\x00\x00':
+            elif aBuf[:4] == b'\xFE\xFF\x00\x00':
                 # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
-                self.result = {'encoding': "X-ISO-10646-UCS-4-3412", 'confidence': 1.0}
-            elif aBuf[:4] == '\x00\x00\xFF\xFE':
+                self.result = {
+                    'encoding': "X-ISO-10646-UCS-4-3412",
+                    'confidence': 1.0
+                }
+            elif aBuf[:4] == b'\x00\x00\xFF\xFE':
                 # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
-                self.result = {'encoding': "X-ISO-10646-UCS-4-2143", 'confidence': 1.0}
-            elif aBuf[:2] == '\xFF\xFE':
+                self.result = {
+                    'encoding': "X-ISO-10646-UCS-4-2143",
+                    'confidence': 1.0
+                }
+            elif aBuf[:2] == codecs.BOM_LE:
                 # FF FE  UTF-16, little endian BOM
                 self.result = {'encoding': "UTF-16LE", 'confidence': 1.0}
-            elif aBuf[:2] == '\xFE\xFF':
+            elif aBuf[:2] == codecs.BOM_BE:
                 # FE FF  UTF-16, big endian BOM
                 self.result = {'encoding': "UTF-16BE", 'confidence': 1.0}
 
-        self._mGotData = constants.True
+        self._mGotData = True
         if self.result['encoding'] and (self.result['confidence'] > 0.0):
-            self.done = constants.True
+            self.done = True
             return
 
         if self._mInputState == ePureAscii:
             if self._highBitDetector.search(aBuf):
                 self._mInputState = eHighbyte
-            elif (self._mInputState == ePureAscii) and self._escDetector.search(self._mLastChar + aBuf):
+            elif ((self._mInputState == ePureAscii) and
+                    self._escDetector.search(self._mLastChar + aBuf)):
                 self._mInputState = eEscAscii
 
-        self._mLastChar = aBuf[-1]
+        self._mLastChar = aBuf[-1:]
 
         if self._mInputState == eEscAscii:
             if not self._mEscCharSetProber:
@@ -107,25 +119,27 @@ class UniversalDetector:
             if self._mEscCharSetProber.feed(aBuf) == constants.eFoundIt:
                 self.result = {'encoding': self._mEscCharSetProber.get_charset_name(),
                                'confidence': self._mEscCharSetProber.get_confidence()}
-                self.done = constants.True
+                self.done = True
         elif self._mInputState == eHighbyte:
             if not self._mCharSetProbers:
-                self._mCharSetProbers = [MBCSGroupProber(), SBCSGroupProber(), Latin1Prober()]
+                self._mCharSetProbers = [MBCSGroupProber(), SBCSGroupProber(),
+                                         Latin1Prober()]
             for prober in self._mCharSetProbers:
                 if prober.feed(aBuf) == constants.eFoundIt:
                     self.result = {'encoding': prober.get_charset_name(),
                                    'confidence': prober.get_confidence()}
-                    self.done = constants.True
+                    self.done = True
                     break
 
     def close(self):
-        if self.done: return
+        if self.done:
+            return
         if not self._mGotData:
             if constants._debug:
                 sys.stderr.write('no data received!\n')
             return
-        self.done = constants.True
-        
+        self.done = True
+
         if self._mInputState == ePureAscii:
             self.result = {'encoding': 'ascii', 'confidence': 1.0}
             return self.result
@@ -135,7 +149,8 @@ class UniversalDetector:
             maxProberConfidence = 0.0
             maxProber = None
             for prober in self._mCharSetProbers:
-                if not prober: continue
+                if not prober:
+                    continue
                 proberConfidence = prober.get_confidence()
                 if proberConfidence > maxProberConfidence:
                     maxProberConfidence = proberConfidence
@@ -148,7 +163,8 @@ class UniversalDetector:
         if constants._debug:
             sys.stderr.write('no probers hit minimum threshhold\n')
             for prober in self._mCharSetProbers[0].mProbers:
-                if not prober: continue
-                sys.stderr.write('%s confidence = %s\n' % \
-                                 (prober.get_charset_name(), \
+                if not prober:
+                    continue
+                sys.stderr.write('%s confidence = %s\n' %
+                                 (prober.get_charset_name(),
                                   prober.get_confidence()))

chardet/utf8prober.py

@@ -13,26 +13,26 @@
 # modify it under the terms of the GNU Lesser General Public
 # License as published by the Free Software Foundation; either
 # version 2.1 of the License, or (at your option) any later version.
-# 
+#
 # This library is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 # Lesser General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU Lesser General Public
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
-import constants, sys
-from constants import eStart, eError, eItsMe
-from charsetprober import CharSetProber
-from codingstatemachine import CodingStateMachine
-from mbcssm import UTF8SMModel
+from . import constants
+from .charsetprober import CharSetProber
+from .codingstatemachine import CodingStateMachine
+from .mbcssm import UTF8SMModel
 
 ONE_CHAR_PROB = 0.5
 
+
 class UTF8Prober(CharSetProber):
     def __init__(self):
         CharSetProber.__init__(self)
@@ -50,13 +50,13 @@ class UTF8Prober(CharSetProber):
     def feed(self, aBuf):
         for c in aBuf:
             codingState = self._mCodingSM.next_state(c)
-            if codingState == eError:
+            if codingState == constants.eError:
                 self._mState = constants.eNotMe
                 break
-            elif codingState == eItsMe:
+            elif codingState == constants.eItsMe:
                 self._mState = constants.eFoundIt
                 break
-            elif codingState == eStart:
+            elif codingState == constants.eStart:
                 if self._mCodingSM.get_current_charlen() >= 2:
                     self._mNumOfMBChar += 1
 

downloader.py

@@ -1,315 +1,302 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team
+# Copyright 2014 Fanficdownloader team
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
+# distributed under the License is distributed on an 'AS IS' BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
 
-import sys, os
-from os.path import normpath, expanduser, isfile, join
-from StringIO import StringIO
-from optparse import OptionParser      
-import getpass
-import string
-import ConfigParser
+from optparse import OptionParser
+from os.path import expanduser, isfile, join
 from subprocess import call
-
+import ConfigParser
+import getpass
 import logging
+import pprint
+import string
+import sys
+
+
+if sys.version_info < (2, 5):
+    print 'This program requires Python 2.5 or newer.'
+    sys.exit(1)
+
 if sys.version_info >= (2, 7):
     # suppresses default logger.  Logging is setup in fanficdownload/__init__.py so it works in calibre, too.
     rootlogger = logging.getLogger()
-    loghandler=logging.NullHandler()
-    loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
+    loghandler = logging.NullHandler()
+    loghandler.setFormatter(logging.Formatter('(=====)(levelname)s:%(message)s'))
     rootlogger.addHandler(loghandler)
 
 try:
-    from calibre.constants import numeric_version as calibre_version
-    is_calibre = True
-except:
-    is_calibre = False
-
-# using try/except directly was masking errors during development.
-if is_calibre:
     # running under calibre
-    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
     from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
     from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
     from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
-else:
-    from fanficdownloader import adapters,writers,exceptions
+except ImportError:
+    from fanficdownloader import adapters, writers, exceptions
     from fanficdownloader.configurable import Configuration
     from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
     from fanficdownloader.geturls import get_urls_from_page
-    
 
-if sys.version_info < (2, 5):
-    print "This program requires Python 2.5 or newer."
-    sys.exit(1)
 
-def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
-    writer = writers.getWriter(writeformat,config,adapter)
-    writer.writeStory(outstream=outstream,metaonly=metaonly)
-    output_filename=writer.getOutputFileName()
+def write_story(config, adapter, writeformat, metaonly=False, outstream=None):
+    writer = writers.getWriter(writeformat, config, adapter)
+    writer.writeStory(outstream=outstream, metaonly=metaonly)
+    output_filename = writer.getOutputFileName()
     del writer
     return output_filename
 
-def main(argv,
-         parser=None,
-         passed_defaultsini=None,
-         passed_personalini=None):
-   # read in args, anything starting with -- will be treated as --<varible>=<value>
-   if not parser:
-       parser = OptionParser("usage: %prog [options] storyurl")
-   parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT")
 
-   if passed_defaultsini:
-       config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini"
-   else:
-       config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini"
-   parser.add_option("-c", "--config",
-                     action="append", dest="configfile", default=None,
-                     help=config_help, metavar="CONFIG")
-   parser.add_option("-b", "--begin", dest="begin", default=None,
-                     help="Begin with Chapter START", metavar="START")
-   parser.add_option("-e", "--end", dest="end", default=None,
-                     help="End with Chapter END", metavar="END")
-   parser.add_option("-o", "--option",
-                     action="append", dest="options",
-                     help="set an option NAME=VALUE", metavar="NAME=VALUE")
-   parser.add_option("-m", "--meta-only",
-                     action="store_true", dest="metaonly",
-                     help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
-   parser.add_option("-u", "--update-epub",
-                     action="store_true", dest="update",
-                     help="Update an existing epub with new chapters, give epub filename instead of storyurl.",)
-   parser.add_option("--update-cover",
-                     action="store_true", dest="updatecover",
-                     help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
-   parser.add_option("--force",
-                     action="store_true", dest="force",
-                     help="Force overwrite of an existing epub, download and overwrite all chapters.",)
-   parser.add_option("-l", "--list",
-                     action="store_true", dest="list",
-                     help="Get list of valid story URLs from page given.",)
-   parser.add_option("-n", "--normalize-list",
-                     action="store_true", dest="normalize",default=False,
-                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
-   parser.add_option("-s", "--sites-list",
-                     action="store_true", dest="siteslist",default=False,
-                     help="Get list of valid story URLs examples.",)
-   parser.add_option("-d", "--debug",
-                     action="store_true", dest="debug",
-                     help="Show debug output while downloading.",)
-   
-   (options, args) = parser.parse_args(argv)
+def main(argv, parser=None, passed_defaultsini=None, passed_personalini=None):
+    # read in args, anything starting with -- will be treated as --<varible>=<value>
+    if not parser:
+        parser = OptionParser('usage: %prog [options] storyurl')
+    parser.add_option('-f', '--format', dest='format', default='epub',
+                      help='write story as FORMAT, epub(default), mobi, text or html', metavar='FORMAT')
 
-   if not options.debug:
-       logger = logging.getLogger("fanficdownloader")
-       logger.setLevel(logging.INFO)
-   
-   if not options.siteslist and len(args) != 1:
-       parser.error("incorrect number of arguments")
+    if passed_defaultsini:
+        config_help = 'read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini'
+    else:
+        config_help = 'read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini'
+    parser.add_option('-c', '--config',
+                      action='append', dest='configfile', default=None,
+                      help=config_help, metavar='CONFIG')
+    parser.add_option('-b', '--begin', dest='begin', default=None,
+                      help='Begin with Chapter START', metavar='START')
+    parser.add_option('-e', '--end', dest='end', default=None,
+                      help='End with Chapter END', metavar='END')
+    parser.add_option('-o', '--option',
+                      action='append', dest='options',
+                      help='set an option NAME=VALUE', metavar='NAME=VALUE')
+    parser.add_option('-m', '--meta-only',
+                      action='store_true', dest='metaonly',
+                      help='Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.', )
+    parser.add_option('-u', '--update-epub',
+                      action='store_true', dest='update',
+                      help='Update an existing epub with new chapters, give epub filename instead of storyurl.', )
+    parser.add_option('--update-cover',
+                      action='store_true', dest='updatecover',
+                      help='Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.', )
+    parser.add_option('--force',
+                      action='store_true', dest='force',
+                      help='Force overwrite of an existing epub, download and overwrite all chapters.', )
+    parser.add_option('-l', '--list',
+                      action='store_true', dest='list',
+                      help='Get list of valid story URLs from page given.', )
+    parser.add_option('-n', '--normalize-list',
+                      action='store_true', dest='normalize', default=False,
+                      help='Get list of valid story URLs from page given, but normalized to standard forms.', )
+    parser.add_option('-s', '--sites-list',
+                      action='store_true', dest='siteslist', default=False,
+                      help='Get list of valid story URLs examples.', )
+    parser.add_option('-d', '--debug',
+                      action='store_true', dest='debug',
+                      help='Show debug output while downloading.', )
 
-   if options.siteslist:
-       for (site,examples) in adapters.getSiteExamples():
-           print("\n====%s====\n\nExample URLs:"%site)
-           for u in examples:
-               print("  * %s"%u)
-       return
+    options, args = parser.parse_args(argv)
 
-   if options.update and options.format != 'epub':
-       parser.error("-u/--update-epub only works with epub")
+    if not options.debug:
+        logger = logging.getLogger('fanficdownloader')
+        logger.setLevel(logging.INFO)
 
-   ## Attempt to update an existing epub.
-   chaptercount = None
-   output_filename = None
-   if options.update:
-       try:
-           (url,chaptercount) = get_dcsource_chaptercount(args[0])
-           if not url:
-               print "No story URL found in epub to update."
-               return
-           print "Updating %s, URL: %s" % (args[0],url)
-           output_filename = args[0]
-       except:
-           # if there's an error reading the update file, maybe it's a URL?
-           # we'll look for an existing outputfile down below.
-           url = args[0]
-   else:
-       url = args[0]
+    if not options.siteslist and len(args) != 1:
+        parser.error('incorrect number of arguments')
 
-   try:
-       configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
-   except exceptions.UnknownSite, e:
-       if options.list or options.normalize:
-           # list for page doesn't have to be a supported site.
-           configuration = Configuration("test1.com",options.format)
-       else:
-           raise e
+    if options.siteslist:
+        for site, examples in adapters.getSiteExamples():
+            print '\n====%s====\n\nExample URLs:' % site
+            for u in examples:
+                print '  * %s' % u
+        return
 
-   conflist = []
-   homepath = join(expanduser("~"),".fanficdownloader")
+    if options.update and options.format != 'epub':
+        parser.error('-u/--update-epub only works with epub')
 
-   if passed_defaultsini:
-       configuration.readfp(passed_defaultsini)
-   
-   if isfile(join(homepath,"defaults.ini")):
-       conflist.append(join(homepath,"defaults.ini"))
-   if isfile("defaults.ini"):
-       conflist.append("defaults.ini")
-       
-   if passed_personalini:
-       configuration.readfp(passed_personalini)
-   
-   if isfile(join(homepath,"personal.ini")):
-       conflist.append(join(homepath,"personal.ini"))
-   if isfile("personal.ini"):
-       conflist.append("personal.ini")
-       
-   if options.configfile:
-       conflist.extend(options.configfile)
-      
-   logging.debug('reading %s config file(s), if present'%conflist)
-   configuration.read(conflist)
+    # Attempt to update an existing epub.
+    chaptercount = None
+    output_filename = None
+    if options.update:
+        try:
+            url, chaptercount = get_dcsource_chaptercount(args[0])
+            if not url:
+                print 'No story URL found in epub to update.'
+                return
+            print 'Updating %s, URL: %s' % (args[0], url)
+            output_filename = args[0]
+        except Exception:
+            # if there's an error reading the update file, maybe it's a URL?
+            # we'll look for an existing outputfile down below.
+            url = args[0]
+    else:
+        url = args[0]
 
-   try:
-       configuration.add_section("overrides")
-   except ConfigParser.DuplicateSectionError:
-       pass
+    try:
+        configuration = Configuration(adapters.getConfigSectionFor(url), options.format)
+    except exceptions.UnknownSite, e:
+        if options.list or options.normalize:
+            # list for page doesn't have to be a supported site.
+            configuration = Configuration('test1.com', options.format)
+        else:
+            raise e
 
-   if options.force:
-       configuration.set("overrides","always_overwrite","true")
+    conflist = []
+    homepath = join(expanduser('~'), '.fanficdownloader')
 
-   if options.update and chaptercount:
-       configuration.set("overrides","output_filename",output_filename)
-       
-   if options.update and not options.updatecover:
-       configuration.set("overrides","never_make_cover","true")
+    if passed_defaultsini:
+        configuration.readfp(passed_defaultsini)
 
-   # images only for epub, even if the user mistakenly turned it
-   # on else where.
-   if options.format not in ("epub","html"):
-       configuration.set("overrides","include_images","false")
-       
-   if options.options:
-       for opt in options.options:
-           (var,val) = opt.split('=')
-           configuration.set("overrides",var,val)
+    if isfile(join(homepath, 'defaults.ini')):
+        conflist.append(join(homepath, 'defaults.ini'))
+    if isfile('defaults.ini'):
+        conflist.append('defaults.ini')
 
-   if options.list or options.normalize:
-       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
-       print "\n".join(retlist)               
-       return
+    if passed_personalini:
+        configuration.readfp(passed_personalini)
 
-   try:
-       adapter = adapters.getAdapter(configuration,url)
-       adapter.setChaptersRange(options.begin,options.end)
+    if isfile(join(homepath, 'personal.ini')):
+        conflist.append(join(homepath, 'personal.ini'))
+    if isfile('personal.ini'):
+        conflist.append('personal.ini')
 
-       # check for updating from URL (vs from file)
-       if options.update and not chaptercount:
-           try:
-               writer = writers.getWriter("epub",configuration,adapter)
-               output_filename=writer.getOutputFileName()
-               (noturl,chaptercount) = get_dcsource_chaptercount(output_filename)
-               print "Updating %s, URL: %s" % (output_filename,url)
-           except:
-               options.update = False
-               pass
-       
-       ## Check for include_images and absence of PIL, give warning.
-       if adapter.getConfig('include_images'):
-           try:
-               from calibre.utils.magick import Image
-               logging.debug("Using calibre.utils.magick")
-           except:
-               try:
-                   import Image
-                   logging.debug("Using PIL")
-               except:
-                   print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
-                   if not sys.stdin.readline().strip().lower().startswith('y'):
-                       return
+    if options.configfile:
+        conflist.extend(options.configfile)
 
-       ## three tries, that's enough if both user/pass & is_adult needed,
-       ## or a couple tries of one or the other
-       for x in range(0,2):
-           try:
-               adapter.getStoryMetadataOnly()
-           except exceptions.FailedToLogin, f:
-               if f.passwdonly:
-                   print "Story requires a password."
-               else:
-                   print "Login Failed, Need Username/Password."
-                   sys.stdout.write("Username: ")
-                   adapter.username = sys.stdin.readline().strip()
-               adapter.password = getpass.getpass(prompt='Password: ')
-               #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
-           except exceptions.AdultCheckRequired:
-               print "Please confirm you are an adult in your locale: (y/n)?"
-               if sys.stdin.readline().strip().lower().startswith('y'):
-                   adapter.is_adult=True
+    logging.debug('reading %s config file(s), if present' % conflist)
+    configuration.read(conflist)
 
-       if options.update and not options.force:
-           urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
+    try:
+        configuration.add_section('overrides')
+    except ConfigParser.DuplicateSectionError:
+        pass
 
-           if chaptercount == urlchaptercount and not options.metaonly:
-               print "%s already contains %d chapters." % (output_filename,chaptercount)
-           elif chaptercount > urlchaptercount:
-               print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
-           elif chaptercount == 0:
-               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
-           else:
-               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
-               if not options.metaonly:
+    if options.force:
+        configuration.set('overrides', 'always_overwrite', 'true')
 
-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(output_filename)
+    if options.update and chaptercount:
+        configuration.set('overrides', 'output_filename', output_filename)
 
-                   writeStory(configuration,adapter,"epub")
-                   
-       else:
-           # regular download
-           if options.metaonly:
-               print adapter.getStoryMetadataOnly()
-           
-           output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
-       
-       if not options.metaonly and adapter.getConfig("post_process_cmd"):
-           metadata = adapter.story.metadata
-           metadata['output_filename']=output_filename
-           call(string.Template(adapter.getConfig("post_process_cmd"))
-                .substitute(metadata), shell=True)
-           
-       del adapter
-   
-   except exceptions.InvalidStoryURL, isu:
-       print isu
-   except exceptions.StoryDoesNotExist, dne:
-       print dne
-   except exceptions.UnknownSite, us:
-       print us
+    if options.update and not options.updatecover:
+        configuration.set('overrides', 'never_make_cover', 'true')
 
-if __name__ == "__main__":
-    #import time
-    #start = time.time()
+    # images only for epub, even if the user mistakenly turned it
+    # on else where.
+    if options.format not in ('epub', 'html'):
+        configuration.set('overrides', 'include_images', 'false')
+
+    if options.options:
+        for opt in options.options:
+            (var, val) = opt.split('=')
+            configuration.set('overrides', var, val)
+
+    if options.list or options.normalize:
+        retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
+        print '\n'.join(retlist)
+        return
+
+    try:
+        adapter = adapters.getAdapter(configuration, url)
+        adapter.setChaptersRange(options.begin, options.end)
+
+        # check for updating from URL (vs from file)
+        if options.update and not chaptercount:
+            try:
+                writer = writers.getWriter('epub', configuration, adapter)
+                output_filename = writer.getOutputFileName()
+                noturl, chaptercount = get_dcsource_chaptercount(output_filename)
+                print 'Updating %s, URL: %s' % (output_filename, url)
+            except Exception:
+                options.update = False
+                pass
+
+        # Check for include_images without no_image_processing. In absence of PIL, give warning.
+        if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
+            try:
+                from calibre.utils.magick import Image
+
+                logging.debug('Using calibre.utils.magick')
+            except ImportError:
+                try:
+                    import Image
+
+                    logging.debug('Using PIL')
+                except ImportError:
+                    print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
+                    if not sys.stdin.readline().strip().lower().startswith('y'):
+                        return
+
+        # three tries, that's enough if both user/pass & is_adult needed,
+        # or a couple tries of one or the other
+        for x in range(0, 2):
+            try:
+                adapter.getStoryMetadataOnly()
+            except exceptions.FailedToLogin, f:
+                if f.passwdonly:
+                    print 'Story requires a password.'
+                else:
+                    print 'Login Failed, Need Username/Password.'
+                    sys.stdout.write('Username: ')
+                    adapter.username = sys.stdin.readline().strip()
+                adapter.password = getpass.getpass(prompt='Password: ')
+                # print('Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
+            except exceptions.AdultCheckRequired:
+                print 'Please confirm you are an adult in your locale: (y/n)?'
+                if sys.stdin.readline().strip().lower().startswith('y'):
+                    adapter.is_adult = True
+
+        if options.update and not options.force:
+            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
+
+            if chaptercount == urlchaptercount and not options.metaonly:
+                print '%s already contains %d chapters.' % (output_filename, chaptercount)
+            elif chaptercount > urlchaptercount:
+                print '%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount)
+            elif chaptercount == 0:
+                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % output_filename
+            else:
+                # update now handled by pre-populating the old
+                # images and chapters in the adapter rather than
+                # merging epubs.
+                url, chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, adapter.logfile = get_update_data(output_filename)
+
+                print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
+
+                if not options.update and chaptercount == urlchaptercount and adapter.getConfig('do_update_hook'):
+                    adapter.hookForUpdates(chaptercount)
+
+                write_story(configuration, adapter, 'epub')
+
+        else:
+            # regular download
+            if options.metaonly:
+                pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
+
+            output_filename = write_story(configuration, adapter, options.format, options.metaonly)
+
+        if not options.metaonly and adapter.getConfig('post_process_cmd'):
+            metadata = adapter.story.metadata
+            metadata['output_filename'] = output_filename
+            call(string.Template(adapter.getConfig('post_process_cmd')).substitute(metadata), shell=True)
+
+        del adapter
+
+    except exceptions.InvalidStoryURL, isu:
+        print isu
+    except exceptions.StoryDoesNotExist, dne:
+        print dne
+    except exceptions.UnknownSite, us:
+        print us
+
+
+if __name__ == '__main__':
     main(sys.argv[1:])
-    #print("Total time seconds:%f"%(time.time()-start))

editconfig.html

@@ -71,7 +71,7 @@
 	Copyright © Fanficdownloader team
       </div>
 
-      <div style="margin-top: 1em; text-align: center'">
+      <div style="margin-top: 1em; text-align: center">
         <script type="text/javascript"><!--
                                           google_ad_client = "pub-2027714004231956";
                                           /* FFD */

example.ini

@@ -53,7 +53,7 @@
 ## default is false
 #collect_series: true
 
-[www.ficwad.com]
+[ficwad.com]
 #username:YourUsername
 #password:YourPassword
 
@@ -68,7 +68,7 @@
 ## confirm they are adult for adult content.
 #is_adult:true
 
-[www.thewriterscoffeeshop.com]
+[www.twcslibrary.net]
 #username:YourName
 #password:yourpassword
 #is_adult:true

fanficdownloader/BeautifulSoup.py

@@ -114,6 +114,9 @@ class PageElement(object):
     """Contains the navigational information for some part of the page
     (either a tag or a piece of text)"""
 
+    ## Added for FFDL.
+    bs3=True
+
     def setup(self, parent=None, previous=None):
         """Sets up the initial relations between this element and
         other elements."""
@@ -679,7 +682,7 @@ class Tag(PageElement):
         return self.__str__(encoding)
 
     def __unicode__(self):
-        return self.__str__(None)
+        return str(self).decode(DEFAULT_OUTPUT_ENCODING)
 
     BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
                                            + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"

fanficdownloader/HtmlTagStack.py

@@ -0,0 +1,57 @@
+# coding: utf-8
+
+import re
+import codecs
+
+stack = []
+
+def get_end_tag(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'.*<([^\ >]+).*', r'</\1>', tag)
+    return u''
+
+def get_tag_name(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'</*([^\ >]+).*', r'\1', tag)
+    return u''
+
+def push(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        stack.append(tag)
+
+def pop():
+    if len(stack) > 0:
+        return stack.pop()
+    return u''
+
+def pop_end_tag():
+    return unicode(get_end_tag(pop()))
+
+def spool_end():
+    html = u''
+    for tag in reversed(stack):
+        html += get_end_tag(tag)
+    return html
+
+def spool_start():
+    html = u''
+    for item in stack:
+        html += item
+    return html
+
+def has_elements():
+    return len(stack) > 0
+
+def get_last():
+    # t = pop()
+    # push(t)
+    # return t
+    if len(stack) > 0:
+        return stack[len(stack)-1]
+    return u''
+
+def flush():
+    del stack[:]
+
+def get_stack():
+    return stack

fanficdownloader/__init__.py

@@ -4,13 +4,16 @@ try:
     # just a way to switch between web service and CLI/PI
     import google.appengine.api 
 except:
-    import sys
-    if sys.version_info >= (2, 7):
-        import logging
-        logger = logging.getLogger(__name__)
-        loghandler=logging.StreamHandler()
-        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-        logger.addHandler(loghandler)
-        loghandler.setLevel(logging.DEBUG)
-        logger.setLevel(logging.DEBUG)
+    try: # just a way to switch between CLI and PI
+        import calibre.constants
+    except:
+        import sys
+        if sys.version_info >= (2, 7):
+            import logging
+            logger = logging.getLogger(__name__)
+            loghandler=logging.StreamHandler()
+            loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+            logger.addHandler(loghandler)
+            loghandler.setLevel(logging.DEBUG)
+            logger.setLevel(logging.DEBUG)
 

fanficdownloader/adapters/__init__.py

@@ -29,17 +29,17 @@ from ..configurable import Configuration
 
 import adapter_test1
 import adapter_fanfictionnet
-import adapter_castlefansorg
+import adapter_fanficcastletvnet
 import adapter_fictionalleyorg
 import adapter_fictionpresscom
 import adapter_ficwadcom
 import adapter_fimfictionnet
 import adapter_harrypotterfanfictioncom
 import adapter_mediaminerorg
-import adapter_potionsandsnitchesnet
+import adapter_potionsandsnitches
 import adapter_tenhawkpresentscom
 import adapter_adastrafanficcom
-import adapter_thewriterscoffeeshopcom
+import adapter_twcslibrarynet
 import adapter_tthfanficorg
 import adapter_twilightednet
 import adapter_twiwritenet
@@ -119,11 +119,33 @@ import adapter_nickandgregnet
 import adapter_potterheadsanonymouscom
 import adapter_simplyundeniablecom
 import adapter_scarheadnet
+import adapter_fictionpadcom
+import adapter_storiesonlinenet
+import adapter_trekiverseorg
+import adapter_literotica
+import adapter_voracity2eficcom
+import adapter_spikeluvercom
+import adapter_bloodshedversecom
+import adapter_nocturnallightnet
+import adapter_fanfichu
+import adapter_fanfictioncsodaidokhu
+import adapter_fictionmaniatv
+import adapter_bdsmgeschichten
+import adapter_tolkienfanfiction
+import adapter_themaplebookshelf
+import adapter_fannation
+import adapter_sheppardweircom
+import adapter_samandjacknet
+import adapter_csiforensicscom
+import adapter_lotrfanfictioncom
+import adapter_fhsarchivecom
+import adapter_fanfictionjunkiesde
+import adapter_devianthearts
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
 ## to pick out the adapter.
-    
+
 ## List of registered site adapters.
 __class_list = []
 __domain_map = {}
@@ -142,6 +164,13 @@ for x in imports():
             __domain_map[site]=cls
 
 def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
     if not getNormalStoryURL.__dummyconfig:
         getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
     # pulling up an adapter is pretty low over-head.  If
@@ -149,10 +178,11 @@ def getNormalStoryURL(url):
     try:
         adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
         url = adapter.url
+        site = adapter.getSiteDomain()
         del adapter
-        return url
+        return (url,site)
     except:
-        return None;
+        return None
 
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
@@ -183,18 +213,20 @@ def getConfigSectionFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
         return cls.getConfigSection()
-    
+
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
 def getClassFor(url):
     ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
+    if fixedurl.startswith("//"):
+        fixedurl = "http:%s"%url
     if not fixedurl.startswith("http"):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.
     fixedurl = re.sub(r"#.*$","",fixedurl)
-    
+
     parsedUrl = up.urlparse(fixedurl)
     domain = parsedUrl.netloc.lower()
     if( domain != parsedUrl.netloc ):
@@ -205,17 +237,17 @@ def getClassFor(url):
         domain = domain.replace("www.","")
         #logger.debug("trying site:without www: "+domain)
         cls = getClassFromList(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
+        fixedurl = re.sub(r"^http(s?)://www\.",r"http\1://",fixedurl)
     if not cls:
         #logger.debug("trying site:www."+domain)
         cls = getClassFromList("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
+        fixedurl = re.sub(r"^http(s?)://",r"http\1://www.",fixedurl)
 
     if cls:
         fixedurl = cls.stripURLParameters(fixedurl)
-    
+
     return (cls,fixedurl)
-    
+
 def getClassFromList(domain):
     try:
         return __domain_map[domain]

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -53,12 +54,19 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         return 'www.adastrafanfic.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
     def extractChapterUrlsAndMetadata(self):
 
         if self.is_adult or self.getConfig("is_adult"):

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -1,6 +1,6 @@
 #  -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team
+# Copyright 2014 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 
-from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
@@ -77,14 +76,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archiveofourown.org'
 
-    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/works/123456 http://"+cls.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+cls.getSiteDomain()+"/works/123456/chapters/78901"
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
         # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -123,6 +122,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         else:
             return True
 
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -145,51 +151,67 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
+                raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
+        if "Sorry, we couldn&#x27;t find the work you were looking for." in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+            
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url,data)
-            data = self._fetchUrl(url)
-            meta = self._fetchUrl(metaurl)
+            data = self._fetchUrl(url,usecache=False)
+            meta = self._fetchUrl(metaurl,usecache=False)
             
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        metasoup = bs.BeautifulSoup(meta)
+        soup = self.make_soup(data)
+        for tag in soup.findAll('div',id='admin-banner'):
+            tag.extract()
+        metasoup = self.make_soup(meta)
+        for tag in metasoup.findAll('div',id='admin-banner'):
+            tag.extract()
 
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
+        a = soup.find('a', href=re.compile(r"/works/\d+$"))
         self.story.setMetadata('title',stripHTML(a))
 		
         # Find authorid and URL from... author url.
-        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        alist = soup.findAll('a', href=re.compile(r"/users/\w+/pseuds/\w+"))
         if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
             self.story.setMetadata('author','Anonymous')
             self.story.setMetadata('authorUrl','http://archiveofourown.org/')
             self.story.setMetadata('authorId','0')
         else:
             for a in alist:
-                self.story.addToList('authorId',a['href'].split('/')[2])
-                self.story.addToList('authorUrl','http://'+self.host+a['href'])
+                self.story.addToList('authorId',a['href'].split('/')[-1])
+                self.story.addToList('authorUrl',a['href'])
                 self.story.addToList('author',a.text)
 
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on AO3 it's
+        # possible for authors to insert new chapters out-of-order or
+        # change the dates of earlier ones by editing them--That WILL
+        # break epub update.
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
         self.story.setMetadata('numChapters',len(chapters))
         logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
-            else:
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
+        else:
+            for index, chapter in enumerate(chapters):
+                # strip just in case there's tags, like <i> in chapter titles.
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-
-
+                # (2013-09-21)
+                date = stripHTML(chapter.findNext('span'))[1:-1]
+                chapterDate = makeDate(date,self.dateformat)
+                if newestChapter == None or chapterDate > newestChapter:
+                    newestChapter = chapterDate
+                    self.newestChapterNum = index
 
         a = metasoup.find('blockquote',{'class':'userstuff'})
         if a != None:
@@ -200,6 +222,10 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if a != None:
             self.story.setMetadata('rating',stripHTML(a.text))
 		
+        d = metasoup.find('dd',{'class':"language"})
+        if d != None:
+            self.story.setMetadata('language',stripHTML(d.text))
+		
         a = metasoup.find('dd',{'class':"fandom tags"})
         fandoms = a.findAll('a',{'class':"tag"})
         for fandom in fandoms:
@@ -299,32 +325,50 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                     self.setSeries(series_name, series_index)
                     self.story.setMetadata('seriesUrl',series_url)
 
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
 		
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        chapter=self.make_soup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
-        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+        soup = self.make_soup(data)
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        def append_tag(elem,tag,string):
+            '''bs4 requires tags be added separately.'''
+            new_tag = soup.new_tag(tag)
+            new_tag.string=string
+            elem.append(new_tag)
+    
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
             if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    append_tag(chapter,'b',"Author's Note:")
+                    chapter.append(headnotes)
         
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                append_tag(chapter,'b',"Summary for the Chapter:")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
             if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    append_tag(chapter,'b',"Notes for the Chapter:")
+                    chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
         chtext = text.find('h3', {'class' : "landmark heading"})
@@ -332,17 +376,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chtext.extract()
         chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                append_tag(chapter,'b',"Notes for the Chapter:")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                append_tag(chapter,'b',"Author's Note:")
+                chapter.append(footnotes)
 			
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -71,7 +71,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         return ['archive.skyehawke.com','www.skyehawke.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 
     def getSiteURLPattern(self):
@@ -91,8 +91,6 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        data = self._fetchUrl(url)
-            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         return 'ashwinder.sycophanthex.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -64,8 +65,8 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
         return 'asr3.slashzone.org'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_bdsmgeschichten.py

@@ -0,0 +1,346 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+def _translate_date_german_english(date):
+    fullmon = {"Januar":"01",
+               "Februar":"02",
+               u"März":"03",
+               "April":"04",
+               "Mai":"05",
+               "Juni":"06",
+               "Juli":"07",
+               "August":"08",
+               "September":"09",
+               "Oktober":"10",
+               "November":"11",
+               "Dezember":"12"}
+    for (name,num) in fullmon.items():
+        date = date.replace(name,num)
+    return date
+
+_REGEX_TRAILING_DIGIT = re.compile("(\d+)$")
+_REGEX_DASH_TO_END = re.compile("-[^-]+$")
+_REGEX_CHAPTER_TITLE = re.compile(ur"""
+    \s*
+    [\u2013-]?
+    \s*
+    ([\dIVX-]+)?
+    \.?
+    \s*
+    [\[\(]?
+    \s*
+    (Teil|Kapitel|Tag)?
+    \s*
+    ([\dIVX-]+)?
+    \s*
+    [\]\)]?
+    \s*
+    $
+""", re.VERBOSE)
+_INITIAL_STEP = 5
+
+class BdsmGeschichtenAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8", "Windows-1252"]
+
+        self.story.setMetadata('siteabbrev','bdsmgesch')
+
+        # Replace possible chapter numbering
+        chapterMatch = _REGEX_TRAILING_DIGIT.search(url)
+        if chapterMatch is None:
+            self.maxChapter = 1
+        else:
+            self.maxChapter = int(chapterMatch.group(1))
+        # url = re.sub(_REGEX_TRAILING_DIGIT, "1", url)
+
+        # set storyId
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
+
+        # normalize URL
+        self._setURL('http://%s/%s' % (self.getSiteDomain(), self.story.getMetadata('storyId')))
+
+        self.dateformat = '%d. %m %Y - %H:%M'
+
+    @staticmethod
+    def getSiteDomain():
+        return 'bdsm-geschichten.net'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.bdsm-geschichten.net', 'www.bdsm-geschichten.net']
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://www.bdsm-geschichten.net/title-of-story-1 http://bdsm-geschichten.net/title-of-story-1"
+
+    def getSiteURLPattern(self):
+        return r"http://(www\.)?bdsm-geschichten.net/(?P<storyId>[a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+
+        try:
+            data1 = self._fetchUrl(self.url)
+            soup = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # Cache the soups so we won't have to redownload in getChapterText later
+        self.soupsCache = {}
+        self.soupsCache[self.url] = soup
+
+        # author
+        authorDiv = soup.find("div", "author-pane-line author-name")
+        authorId = authorDiv.string.strip()
+        self.story.setMetadata('authorId', authorId)
+        self.story.setMetadata('author', authorId)
+        # TODO not really true need to be loggedin for this to work or fetch userid
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId)
+
+        # TODO better metadata
+        date = soup.find("div", {"class": "submitted"}).string.strip()
+        date = re.sub(" &#151;.*", "", date)
+        date = _translate_date_german_english(date)
+        self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+        title1 = soup.find("h1", {'class': 'title'}).string
+
+
+        for tagLink in soup.find("ul", "taxonomy").findAll("a"):
+            self.story.addToList('category', tagLink.string)
+
+        ## Retrieve chapter soups
+        if self.getConfig('find_chapters') == 'guess':
+            self.chapterUrls = []
+            self._find_chapters_by_guessing(title1)
+        else:
+            self._find_chapters_by_parsing(soup)
+
+        firstChapterUrl = self.chapterUrls[0][1]
+        if firstChapterUrl in self.soupsCache:
+            firstChapterSoup = self.soupsCache[firstChapterUrl]
+            h1 = firstChapterSoup.find("h1").text
+        else:
+            h1 = soup.find("h1").text
+
+        h1 = re.sub(_REGEX_CHAPTER_TITLE, "", h1)
+        self.story.setMetadata('title', h1)
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+        return
+
+    def _find_chapters_by_parsing(self, soup):
+
+        # store original soup
+        origSoup = soup
+
+        #
+        # find first chapter
+        #
+        firstLink = None
+        firstLinkDiv = soup.find("div", "field-field-erster-teil")
+        if firstLinkDiv is not None:
+            firstLink = "http://%s%s" % (self.getSiteDomain(), firstLinkDiv.findNext("a")['href'])
+            logger.debug("Found first chapter right away <%s>" % firstLink)
+            try:
+                soup = bs.BeautifulSoup(self._fetchUrl(firstLink))
+                self.soupsCache[firstLink] = soup
+                self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(self.url)
+                else:
+                    raise exceptions.StoryDoesNotExist(firstLink)
+        else:
+            logger.debug("DIDN'T find first chapter right away")
+            # parse previous Link until first
+            while True:
+                prevLink = None
+                prevLinkDiv = soup.find("div", "field-field-vorheriger-teil")
+                if prevLinkDiv is not None:
+                    prevLink = prevLinkDiv.find("a")
+                if prevLink is None:
+                    prevLink = soup.find("a", text=re.compile("&lt;&lt;&lt;")) # <<<
+                if prevLink is None:
+                    logger.debug("Couldn't find prev part")
+                    break
+                else:
+                    logger.debug("Previous Chapter <%s>" % prevLink)
+                    if type(prevLink) != bs.Tag or prevLink.name != "a":
+                        prevLink = prevLink.findParent("a")
+                    if prevLink is None or '#' in prevLink['href']:
+                        logger.debug("Couldn't find prev part (false positive) <%s>" % prevLink)
+                        break
+                    prevLink = prevLink['href']
+                try:
+                    soup = bs.BeautifulSoup(self._fetchUrl(prevLink))
+                    self.soupsCache[prevLink] = soup
+                    prevTtitle = soup.find("h1", {'class': 'title'}).string
+                    self.chapterUrls.insert(0, (prevTtitle, prevLink))
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(nextLink)
+                    else:
+                        raise e
+                firstLink = prevLink
+
+        # if first chapter couldn't be determined, assume the URL originally
+        # passed is the first chapter
+        if firstLink is None:
+            logger.debug("Couldn't set first chapter")
+            firstLink = self.url
+            self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
+
+        # set first URL
+        logger.debug("Set first link: %s" % firstLink)
+        self._setURL(firstLink)
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(firstLink).group('storyId'))
+
+        #
+        # Parse next chapters
+        #
+        while True:
+            nextLink = None
+            nextLinkDiv = soup.find("div", "field-field-naechster-teil")
+            if nextLinkDiv is not None:
+                nextLink = nextLinkDiv.find("a")
+            if nextLink is None:
+                nextLink = soup.find("a", text=re.compile("&gt;&gt;&gt;"))
+            if nextLink is None:
+                nextLink = soup.find("a", text=re.compile("Fortsetzung"))
+
+            if nextLink is None:
+                logger.debug("Couldn't find next part")
+                break
+            else:
+                if type(nextLink) != bs.Tag or nextLink.name != "a":
+                    nextLink = nextLink.findParent("a")
+                if nextLink is None or '#' in nextLink['href']:
+                    logger.debug("Couldn't find next part (false positive) <%s>" % nextLink)
+                    break
+                nextLink = nextLink['href']
+
+            if not nextLink.startswith('http:'):
+                nextLink = 'http://' + self.getSiteDomain() + nextLink
+
+            for loadedChapter in self.chapterUrls:
+                if loadedChapter[0] == nextLink:
+                    logger.debug("ERROR: Repeating chapter <%s> Try to fix it" % nextLink)
+                    nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink)
+                    if nextLinkMatch is not None:
+                        curChap = nextLinkMatch.group(1)
+                        nextLink = re.sub(_REGEX_TRAILING_DIGIT, str(int(curChap) + 1), nextLink)
+                    else:
+                        break
+            try:
+                data = self._fetchUrl(nextLink)
+                soup = bs.BeautifulSoup(data)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(nextLink)
+                else:
+                    raise e
+            title2 = soup.find("h1", {'class': 'title'}).string
+            self.chapterUrls.append((title2, nextLink))
+            logger.debug("Grabbing next chapter URL " + nextLink)
+            self.soupsCache[nextLink] = soup
+            # [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        logger.debug("Chapters: %s" % self.chapterUrls)
+
+
+    def _find_chapters_by_guessing(self, title1):
+        step = _INITIAL_STEP
+        curMax = self.maxChapter + step
+        lastHit = True
+        while True:
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(curMax), self.url) 
+            if nextChapterUrl == self.url:
+                logger.debug("Unable to guess next chapter because URL doesn't end in numbers")
+                break;
+            try:
+                logger.debug("Trying chapter URL " + nextChapterUrl)
+                data = self._fetchUrl(nextChapterUrl)
+                hit = True
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    hit = False
+                else:
+                    raise e
+            if hit:
+                logger.debug("Found chapter URL " + nextChapterUrl)
+                self.maxChapter = curMax
+                self.soupsCache[nextChapterUrl] = bs.BeautifulSoup(data)
+                if not lastHit:
+                    break
+                lastHit = curMax
+                curMax += step
+            else:
+                lastHit = False
+                curMax -= 1
+            logger.debug(curMax)
+
+        for i in xrange(1, self.maxChapter):
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(i), self.url)
+            nextChapterTitle = re.sub("1", str(i), title1)
+            self.chapterUrls.append((nextChapterTitle, nextChapterUrl))
+
+    def getChapterText(self, url):
+
+        if url in self.soupsCache:
+            logger.debug('Getting chapter <%s> from cache' % url)
+            soup = self.soupsCache[url]
+        else:
+            logger.debug('Downloading chapter <%s>' % url)
+            data1 = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data1)
+            #strip comments from soup
+            [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        storyDiv1 = bs.Tag(soup, "div")
+        for para in soup.find("div", "full-node").find('div', 'content').findAll("p"):
+            storyDiv1.append(para)
+        storyDiv1.append('<br />')
+        storytext = self.utf8FromSoup(url,storyDiv1)
+
+        return storytext
+
+
+def getClass():
+    return BdsmGeschichtenAdapter

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -0,0 +1,193 @@
+from datetime import timedelta
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+from ..htmlcleanup import stripHTML
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return BloodshedverseComAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class BloodshedverseComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'bvc'
+    SITE_DOMAIN = 'bloodshedverse.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
+
+    STARTED_DATETIME_FORMAT = '%m/%d/%Y'
+    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_no = query_data['no'][0]
+
+        self.story.setMetadata('storyId', story_no)
+        self._setURL(self.READ_URL_TEMPLATE % story_no)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return BloodshedverseComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL + 'stories.php?go=') + r'(read|chapters)\&no=\d+$'
+
+    # Override stripURLParameters so the "no" parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if stripHTML(soup.title) == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        for option in soup.find('select', {'name': 'chapter'}):
+            title = stripHTML(option)
+            url = self.READ_URL_TEMPLATE % option['value']
+            self.chapterUrls.append((title, url))
+
+        # Get the URL to the author's page and find the correct story entry to
+        # scrape the metadata
+        author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_no = self.story.getMetadata('storyId')
+        # Ignore first list_box div, it only contains the author information
+        for list_box in soup('div', {'class': 'list_box'})[1:]:
+            url = list_box.find('a', {'class': 'fictitle'})['href']
+            query_data = _get_query_data(url)
+
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if query_data['no'] == story_no:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title_anchor = list_box.find('a', {'class': 'fictitle'})
+        self.story.setMetadata('title', stripHTML(title_anchor))
+
+        author_anchor = title_anchor.findNextSibling('a')
+        self.story.setMetadata('author', stripHTML(author_anchor))
+        self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
+
+        list_review = list_box.find('div', {'class': 'list_review'})
+        reviews = stripHTML(list_review.a).split(' ', 1)[0]
+        self.story.setMetadata('reviews', reviews)
+
+        summary_div = list_box.find('div', {'class': 'list_summary'})
+        if not self.getConfig('keep_summary_html'):
+            summary = ''.join(summary_div(text=True))
+        else:
+            summary = self.utf8FromSoup(author_url, summary_div)
+
+        self.story.setMetadata('description', summary)
+
+        # I'm assuming this to be the category, not sure what else it could be
+        first_listinfo = list_box.find('div', {'class': 'list_info'})
+        self.story.addToList('category', stripHTML(first_listinfo.a))
+
+        for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
+            for b_tag in list_info('b'):
+                key = b_tag.string.strip(': ')
+                # Strip colons from the beginning, superfluous spaces and minus
+                # characters from the end, and possibly trailing commas from
+                # the warnings if only one is present
+                value = b_tag.nextSibling.string.strip(': -,')
+
+                if key == 'Genre':
+                    for genre in value.split(', '):
+                        # Ignore the "none" genre
+                        if not genre == 'none':
+                            self.story.addToList('genre', genre)
+
+                elif key == 'Rating':
+                    self.story.setMetadata('rating', value)
+
+                elif key == 'Complete':
+                    self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+                elif key == 'Warning':
+                    for warning in value.split(', '):
+                        # The string here starts with ", " before the actual list
+                        # of values sometimes, so check for an empty warning
+                        # and ignore the "none" warning.
+                        if not warning or warning == 'none':
+                            continue
+
+                        self.story.addToList('warnings', warning)
+
+                elif key == 'Chapters':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Words':
+                    # Apparently only numChapters need to be an integer for
+                    # some strange reason. Remove possible ',' characters as to
+                    # not confuse the codebase down the line
+                    self.story.setMetadata('numWords', value.replace(',', ''))
+
+                elif key == 'Started':
+                    self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
+
+                elif key == 'Updated':
+                    date_string, period = value.rsplit(' ', 1)
+                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
+
+                    # Rather ugly hack to work around Calibre's changing of
+                    # Python's locale setting, causing am/pm to not be properly
+                    # parsed by strptime() when using a non-english locale
+                    if period == 'pm':
+                        date += timedelta(hours=12)
+                    self.story.setMetadata('dateUpdated', date)
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        storytext_div = soup.find('div', {'class': 'storytext'})
+
+        if self.getConfig('strip_text_links'):
+            for anchor in storytext_div('a', {'class': 'FAtxtL'}):
+                navigable_string = BeautifulSoup.NavigableString(anchor.string)
+                anchor.replaceWith(navigable_string)
+
+        return self.utf8FromSoup(url, storytext_div)

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -89,8 +90,8 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         return 'bloodties-fans.com' # XXX
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -87,11 +87,11 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
     
     def setHeader(self):
         "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
         
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://buffynfaith.net/fanfictions/index.php?act=ovr&id=1234 http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=ovr&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234&ch=2"
 
     def getSiteURLPattern(self):
         #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
@@ -101,6 +101,13 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
             r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
         return p
 
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
     def extractChapterUrlsAndMetadata(self):
 
         dateformat = "%d %B %Y"
@@ -109,7 +116,6 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
         
         #set a cookie to get past adult check
         if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
             cookie = cl.Cookie(version=0, name='my_age', value='yes',
                                port=None, port_specified=False,
                                domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -121,8 +127,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
                                comment_url=None,
                                rest={'HttpOnly': None},
                                rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)        
+            self.cookiejar.set_cookie(cookie)
             self.setHeader()
 
         try:

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         return 'chaos.sycophanthex.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -65,8 +65,8 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         return 'www.checkmated.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?story=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story.php?story=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/story.php?story=")+r"\d+$"

fanficdownloader/adapters/adapter_csiforensicscom.py

@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return CSIForensicsComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class CSIForensicsComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','csiforensics')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'csi-forensics.com'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5&skin=elegantcsi"
+        else:
+            addurl="&skin=elegantcsi"
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+				
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "This story is rated NC-17, and therefore is not suitable for minors.  If you are below the age required to view such material in your locality, please return from whence you came." in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+       
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+       
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        smalldiv = soup.find('div', {'class' : 'small'})
+        
+      
+        chars = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+            
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress') 
+        
+        word=soup.find(text=re.compile("Word count:")).split(':')
+        self.story.setMetadata('numWords', word[1])
+
+        cats = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        warnings = smalldiv.findAll('a',href=re.compile(r'browse.php\?type=class(&amp;)type_id=2(&amp;)classid=\d+'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)            
+            			
+        date=soup.find('div',{'class' : 'bottom'})
+        pd=date.find(text=re.compile("Published:")).string.split(': ')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(pd[1].split(' U')[0]), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(pd[2]), self.dateformat))
+		
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        pub=0
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Genres' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:                
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        smalldiv.extract()
+        
+        # Summary  
+        summary = soup.find('div', {'class' : 'content'})
+        self.setDescription(url,summary)        
+           
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
+ 

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -70,8 +71,8 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         return ['www.dark-solace.org','dark-solace.org']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
@@ -94,11 +95,10 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         else:
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
         params['action'] = 'login'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
@@ -183,6 +183,11 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         # first a tag in pagetitle is title
         self.story.setMetadata('title',stripHTML(div.find('a')))
+        div.find('a').extract()
+        # only thing left in div(pagetitle) now should be 'by' and rating.
+        rating = stripHTML(div)
+        if '[' in rating:
+            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
 
         for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
                                                        self.story.getMetadata('storyId')+'&chapter=\d+')):
@@ -235,31 +240,28 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
                 if 'Yes' in value:

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         return 'www.destinysgateway.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_devianthearts.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class DeviantHeartsAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'devianthearts.com'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'devhrt'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%m/%d/%y" 
+   
+    # def handleMetadataPair(self, key, value):
+    #     if key == 'Warnings':
+    #         for val in re.split("\s*,\s*", value):
+    #             if value == 'None':
+    #                 return
+    #             else:
+    #                 # toss numbers only.
+    #                 self.story.addToList('warnings', filter(lambda x : not x.isdigit() , val))
+     
+    # #     elif 'Categories' in key:
+    # #         for val in re.split("\s*>\s*", value):
+    # #             self.story.addToList('category', val)
+    #     else:
+    #         super(FHSArchiveComAdapter, self).handleMetadataPair(key, value) 
+                           
+def getClass():
+    return DeviantHeartsAdapter
+    

fanficdownloader/adapters/adapter_dokugacom.py

@@ -76,8 +76,8 @@ class DokugaComAdapter(BaseSiteAdapter):
         return 'www.dokuga.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfiction/story/1234/1 http://"+cls.getSiteDomain()+"/spark/story/1234/1"
 
     def getSiteURLPattern(self):
         return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -66,8 +66,8 @@ class DotMoonNetAdapter(BaseSiteAdapter):
         return 'www.dotmoon.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/library_view.php?storyid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/library_view.php?storyid=")+r"\d+$"

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         return 'www.dracoandginny.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         return 'dramione.org'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,8 +67,8 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
         return 'efiction.esteliel.de'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -66,15 +67,16 @@ class EFPFanFicNet(BaseSiteAdapter):
         return 'www.efpfanfic.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
-        if 'Fai il login e leggi la storia!' in data:
+        if( 'Fai il login e leggi la storia!' in data or
+            'Questa storia presenta contenuti non adatti ai minori' in data ):
             return True
         else:
             return False
@@ -203,8 +205,8 @@ class EFPFanFicNet(BaseSiteAdapter):
         noteblock = storyblock.find('div', {'class':'notebloc'})
         #print("%s"%noteblock)
         notetext = ("%s" % noteblock).replace("<br />"," |")
-        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />
-        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />
+        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />
+        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />
         # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>
 
         cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -67,8 +68,8 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         return 'erosnsappho.sycophanthex.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_fanficcastletvnet.py

@@ -0,0 +1,322 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return FanficCastleTVNetAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','csltv') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfic.castletv.net' # XXX
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+         ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        
+        # Reviews
+        reviewdata = soup.find('div', {'id' : 'sort'})
+        a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
+        self.story.setMetadata('reviews',stripHTML(a)) 
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfichu.py

@@ -0,0 +1,185 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanficHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class FanficHuAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'ffh'
+    SITE_DOMAIN = 'fanfic.hu'
+    SITE_LANGUAGE = 'Hungarian'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/merengo/'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%s'
+
+    DATE_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanficHuAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&i=1')
+
+        if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        chapter_options = soup.find('form', action='viewstory.php').select('option')
+        # Remove redundant "Fejezetek" option
+        chapter_options.pop(0)
+
+        # If there is still more than one entry remove chapter overview entry
+        if len(chapter_options) > 1:
+            chapter_options.pop(0)
+
+        for option in chapter_options:
+            url = urlparse.urljoin(self.url, option['value'])
+            self.chapterUrls.append((option.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_id = self.story.getMetadata('storyId')
+        for table in soup('table', {'class': 'mainnav'}):
+            title_anchor = table.find('span', {'class': 'storytitle'}).a
+            href = title_anchor['href']
+            if href.startswith('javascript:'):
+                href = href.rsplit(' ', 1)[1].strip("'")
+            query_data = _get_query_data(href)
+
+            if query_data['sid'] == story_id:
+                break
+        else:
+            # This should never happen, the story must be found on the author's
+            # page.
+            raise exceptions.FailedToDownload(self.url)
+
+        self.story.setMetadata('title', title_anchor.string)
+
+        rows = table('tr')
+
+        anchors = rows[0].div('a')
+        author_anchor = anchors[1]
+        query_data = _get_query_data(author_anchor['href'])
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.BASE_URL, author_anchor['href']))
+        self.story.setMetadata('reviews', anchors[3].string)
+
+        if self.getConfig('keep_summary_html'):
+            self.story.setMetadata('description', self.utf8FromSoup(author_url, rows[1].td))
+        else:
+            self.story.setMetadata('description', ''.join(rows[1].td(text=True)))
+
+        for row in rows[3:]:
+            index = 0
+            cells = row('td')
+
+            while index < len(cells):
+                cell = cells[index]
+                key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
+                try:
+                    value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
+                except AttributeError:
+                    value = None
+
+                if key == 'Kategória':
+                    for anchor in cells[index+1]('a'):
+                        self.story.addToList('category', anchor.string)
+
+                elif key == 'Szereplõk':
+                    if cells[index+1].string:
+                        for name in cells[index+1].string.split(', '):
+                            self.story.addToList('character', name)
+
+                elif key == 'Korhatár':
+                    if value != 'nem korhatáros':
+                        self.story.setMetadata('rating', value)
+
+                elif key == 'Figyelmeztetések':
+                    for b_tag in cells[index+1]('b'):
+                        self.story.addToList('warnings', b_tag.string)
+
+                elif key == 'Jellemzõk':
+                    for genre in cells[index+1].string.split(', '):
+                        self.story.addToList('genre', genre)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Megjelenés':
+                    self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Frissítés':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Szavak':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                index += 2
+
+        if self.story.getMetadata('rating') == '18':
+            if not (self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        story_cell = soup.find('form', action='viewstory.php').parent.parent
+
+        for div in story_cell('div'):
+            div.extract()
+
+        return self.utf8FromSoup(url, story_cell)

fanficdownloader/adapters/adapter_fanfictioncsodaidokhu.py

@@ -0,0 +1,218 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanfictionCsodaidokHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FanfictionCsodaidokHuAdapter(BaseSiteAdapter):
+    _SITE_DOMAIN = 'fanfiction.csodaidok.hu'
+    _BASE_URL = 'http://' + _SITE_DOMAIN + '/'
+    _VIEW_STORY_URL_TEMPLATE = _BASE_URL + 'viewstory.php?sid=%s'
+    _VIEW_CHAPTER_URL_TEMPLATE = _VIEW_STORY_URL_TEMPLATE + '&chapter=%s'
+
+    _STORY_DOES_NOT_EXIST_PAGE_TITLE = 'Cím:  Szerző:'
+    _DATE_FORMAT = '%Y.%m.%d'
+    _SITE_LANGUAGE = 'Hungarian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self._VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self._SITE_DOMAIN)
+        self.story.setMetadata('language', self._SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanfictionCsodaidokHuAdapter._SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls._VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self._VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&chapter=1')
+
+        element = soup.find('div', id='pagetitle')
+        page_title = ''.join(element(text=True)).encode(_SOURCE_CODE_ENCODING)
+        if page_title == self._STORY_DOES_NOT_EXIST_PAGE_TITLE:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        author_url = urlparse.urljoin(self.url, element.a['href'])
+
+        story_id = self.story.getMetadata('storyId')
+        element = soup.find('select', {'name': 'chapter'})
+        if element:
+            for option in element('option'):
+                title = option.string
+                url = self._VIEW_CHAPTER_URL_TEMPLATE % (story_id, option['value'])
+                self.chapterUrls.append((title, url))
+
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+
+        for listbox_div in soup('div', {'class': lambda klass: klass and 'listbox' in klass}):
+            a = listbox_div.div.a
+            if not a['href'].startswith('viewstory.php?sid='):
+                continue
+
+            query_data = _get_query_data(a['href'])
+            if query_data['sid'] == story_id:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = ''.join(a(text=True))
+        self.story.setMetadata('title', title)
+        if not self.chapterUrls:
+            self.chapterUrls.append((title, self.url))
+
+        element = a.findNextSibling('a')
+        self.story.setMetadata('author', element.string)
+        query_data = _get_query_data(element['href'])
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', author_url)
+
+        element = element.findNextSibling('span')
+        rating = element.nextSibling.strip(' [')
+
+        if rating.encode(_SOURCE_CODE_ENCODING) != 'Korhatár nélkül':
+            self.story.setMetadata('rating', rating)
+
+        if rating == '18':
+            raise exceptions.AdultCheckRequired(self.url)
+
+        element = element.findNextSiblings('a')[1]
+        self.story.setMetadata('reviews', element.string)
+
+        sections = listbox_div('div', {'class': lambda klass: klass and klass in ['content', 'tail']})
+        for section in sections:
+            for element in section('span', {'class': 'classification'}):
+                key = element.string.encode(_SOURCE_CODE_ENCODING).strip(' :')
+                try:
+                    value = element.nextSibling.string.encode(_SOURCE_CODE_ENCODING).strip()
+                except AttributeError:
+                    value = None
+
+                if key == 'Tartalom':
+                    contents = []
+                    keep_summary_html = self.getConfig('keep_summary_html')
+
+                    for sibling in _yield_next_siblings(element):
+                        if isinstance(sibling, BeautifulSoup.Tag):
+                            if sibling.name == 'span' and sibling.get('class', None) == 'classification':
+                                break
+
+                            if keep_summary_html:
+                                contents.append(self.utf8FromSoup(author_url, sibling))
+                            else:
+                                contents.append(''.join(sibling(text=True)))
+                        else:
+                            contents.append(sibling)
+                    self.story.setMetadata('description', ''.join(contents))
+
+                elif key == 'Kategória':
+                    for sibling in element.findNextSiblings(['a', 'span']):
+                        if sibling.name == 'span':
+                            break
+
+                        self.story.addToList('category', sibling.string)
+
+                elif key == 'Szereplők':
+                    for name in value.split(', '):
+                        self.story.addToList('characters', name)
+
+                elif key == 'Műfaj':
+                    if value != 'Nincs':
+                        self.story.setMetadata('genre', value)
+
+                elif key == 'Figyelmeztetés':
+                    if value != 'Nincs':
+                        for warning in value.split(', '):
+                            self.story.addToList('warnings', warning)
+
+                elif key == 'Kihívás':
+                    if value != 'Nincs':
+                        self.story.setMetadata('challenge', value)
+
+                elif key == 'Sorozat':
+                    if value != 'Nincs':
+                        self.story.setMetadata('series', value)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                elif key == 'Szavak száma':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Feltöltve':
+                    self.story.setMetadata('datePublished', makeDate(value, self._DATE_FORMAT))
+
+                elif key == 'Frissítve':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self._DATE_FORMAT))
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        contents = []
+
+        notes_div = soup.find('div', id='notes')
+        if notes_div:
+            contents.append(self.utf8FromSoup(url, notes_div))
+            story_div = notes_div.findNextSibling('div')
+        else:
+            element = soup.find('div', {'class': 'jumpmenu'})
+            story_div = element.findNextSibling('div')
+
+        contents.append(self.utf8FromSoup(url, story_div.span))
+        return ''.join(contents)

fanficdownloader/adapters/adapter_fanfictionjunkiesde.py

@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return FanfictionJunkiesDeAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FanfictionJunkiesDeAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/efiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ffjde') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfiction-junkies.de' # XXX
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/efiction/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=1" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "For adults only " in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('h4')
+        ## Title      
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/efiction/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        
+        # Reviews
+        reviewdata = soup.find('div', {'id' : 'sort'})
+        a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
+        self.story.setMetadata('reviews',stripHTML(a)) 
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                           
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        list = soup.find('div', {'class':'listbox'})
+        
+        
+        labels = list.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Zusammenfassung' in label:
+                self.setDescription(url,value)
+
+            if 'Eingestuft' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'W&ouml;rter' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Kategorie' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Charaktere' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Abgeschlossen' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Ver&ouml;ffentlicht' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Aktualisiert' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/efiction/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)