From 2b2e7d52ec2e88ab97742d17bf1e6857fa994827 Mon Sep 17 00:00:00 2001 From: sigizmund Date: Sat, 17 Jul 2010 14:44:14 +0100 Subject: [PATCH] Starting to use source control again after a long break :-) --- BeautifulSoup.py | 747 +++++++++++++++++++++++++++++++++-------------- constants.py | 3 +- ffnet.py | 47 ++- ficwad.py | 6 +- html2text.py | 450 ++++++++++++++++++++++++++++ output.py | 30 +- twilighted.py | 134 +++++++++ 7 files changed, 1173 insertions(+), 244 deletions(-) create mode 100644 html2text.py create mode 100644 twilighted.py diff --git a/BeautifulSoup.py b/BeautifulSoup.py index 458f08a..748e6fe 100644 --- a/BeautifulSoup.py +++ b/BeautifulSoup.py @@ -1,7 +1,6 @@ """Beautiful Soup Elixir and Tonic "The Screen-Scraper's Friend" -v3.0.0 http://www.crummy.com/software/BeautifulSoup/ Beautiful Soup parses a (possibly invalid) XML or HTML document into a @@ -25,7 +24,7 @@ if you also install these three packages: http://cjkpython.i18n.org/ Beautiful Soup defines classes for two main parsing strategies: - + * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific language that kind of looks like XML. @@ -41,36 +40,83 @@ For more than you ever wanted to know about Beautiful Soup, see the documentation: http://www.crummy.com/software/BeautifulSoup/documentation.html +Here, have some legalese: + +Copyright (c) 2004-2010, Leonard Richardson + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the the Beautiful Soup Consortium and All + Night Kosher Bakery nor the names of its contributors may be + used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT. + """ from __future__ import generators __author__ = "Leonard Richardson (leonardr@segfault.org)" -__version__ = "3.0.0" -__date__ = "$Date: 2004/10/18 00:14:20 $" -__copyright__ = "Copyright (c) 2004-2005 Leonard Richardson" -__license__ = "PSF" +__version__ = "3.0.8.1" +__copyright__ = "Copyright (c) 2004-2010 Leonard Richardson" +__license__ = "New-style BSD" from sgmllib import SGMLParser, SGMLParseError import codecs +import markupbase import types import re import sgmllib -from htmlentitydefs import name2codepoint +try: + from htmlentitydefs import name2codepoint +except ImportError: + name2codepoint = {} +try: + set +except NameError: + from sets import Set as set -#This code makes Beautiful Soup able to parse XML with namespaces +#These hacks make Beautiful Soup able to parse XML with namespaces sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*') +markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match DEFAULT_OUTPUT_ENCODING = "utf-8" +def _match_css_class(str): + """Build a RE to match the given CSS class.""" + return re.compile(r"(^|.*\s)%s($|\s)" % str) + # First, the classes that represent markup elements. -class PageElement: +class PageElement(object): """Contains the navigational information for some part of the page (either a tag or a piece of text)""" def setup(self, parent=None, previous=None): """Sets up the initial relations between this element and - other elements.""" + other elements.""" self.parent = parent self.previous = previous self.next = None @@ -80,31 +126,41 @@ class PageElement: self.previousSibling = self.parent.contents[-1] self.previousSibling.nextSibling = self - def replaceWith(self, replaceWith): + def replaceWith(self, replaceWith): oldParent = self.parent - myIndex = self.parent.contents.index(self) - if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent: + myIndex = self.parent.index(self) + if hasattr(replaceWith, "parent")\ + and replaceWith.parent is self.parent: # We're replacing this element with one of its siblings. - index = self.parent.contents.index(replaceWith) + index = replaceWith.parent.index(replaceWith) if index and index < myIndex: # Furthermore, it comes before this element. That # means that when we extract it, the index of this # element will change. myIndex = myIndex - 1 - self.extract() + self.extract() oldParent.insert(myIndex, replaceWith) - + + def replaceWithChildren(self): + myParent = self.parent + myIndex = self.parent.index(self) + self.extract() + reversedChildren = list(self.contents) + reversedChildren.reverse() + for child in reversedChildren: + myParent.insert(myIndex, child) + def extract(self): - """Destructively rips this element out of the tree.""" + """Destructively rips this element out of the tree.""" if self.parent: try: - self.parent.contents.remove(self) + del self.parent.contents[self.parent.index(self)] except ValueError: pass #Find the two elements that would be next to each other if #this element (and any children) hadn't been parsed. Connect - #the two. + #the two. lastChild = self._lastRecursiveChild() nextElement = lastChild.next @@ -115,12 +171,13 @@ class PageElement: self.previous = None lastChild.next = None - self.parent = None + self.parent = None if self.previousSibling: self.previousSibling.nextSibling = self.nextSibling if self.nextSibling: self.nextSibling.previousSibling = self.previousSibling - self.previousSibling = self.nextSibling = None + self.previousSibling = self.nextSibling = None + return self def _lastRecursiveChild(self): "Finds the last element beneath this object to be parsed." @@ -130,25 +187,24 @@ class PageElement: return lastChild def insert(self, position, newChild): - if (isinstance(newChild, basestring) - or isinstance(newChild, unicode)) \ + if isinstance(newChild, basestring) \ and not isinstance(newChild, NavigableString): - newChild = NavigableString(newChild) + newChild = NavigableString(newChild) position = min(position, len(self.contents)) - if hasattr(newChild, 'parent') and newChild.parent != None: + if hasattr(newChild, 'parent') and newChild.parent is not None: # We're 'inserting' an element that's already one - # of this object's children. - if newChild.parent == self: - index = self.find(newChild) - if index and index < position: + # of this object's children. + if newChild.parent is self: + index = self.index(newChild) + if index > position: # Furthermore we're moving it further down the # list of this object's children. That means that # when we extract this element, our target index # will jump down one. position = position - 1 newChild.extract() - + newChild.parent = self previousChild = None if position == 0: @@ -160,13 +216,13 @@ class PageElement: newChild.previousSibling.nextSibling = newChild newChild.previous = previousChild._lastRecursiveChild() if newChild.previous: - newChild.previous.next = newChild + newChild.previous.next = newChild newChildsLastElement = newChild._lastRecursiveChild() if position >= len(self.contents): newChild.nextSibling = None - + parent = self parentsNextSibling = None while not parentsNextSibling: @@ -179,8 +235,8 @@ class PageElement: else: newChildsLastElement.next = None else: - nextChild = self.contents[position] - newChild.nextSibling = nextChild + nextChild = self.contents[position] + newChild.nextSibling = nextChild if newChild.nextSibling: newChild.nextSibling.previousSibling = newChild newChildsLastElement.next = nextChild @@ -189,6 +245,10 @@ class PageElement: newChildsLastElement.next.previous = newChildsLastElement self.contents.insert(position, newChild) + def append(self, tag): + """Appends the given tag to the contents of this tag.""" + self.insert(len(self.contents), tag) + def findNext(self, name=None, attrs={}, text=None, **kwargs): """Returns the first item that matches the given criteria and appears after this Tag in the document.""" @@ -197,8 +257,9 @@ class PageElement: def findAllNext(self, name=None, attrs={}, text=None, limit=None, **kwargs): """Returns all items that match the given criteria and appear - before after Tag in the document.""" - return self._findAll(name, attrs, text, limit, self.nextGenerator) + after this Tag in the document.""" + return self._findAll(name, attrs, text, limit, self.nextGenerator, + **kwargs) def findNextSibling(self, name=None, attrs={}, text=None, **kwargs): """Returns the closest sibling to this Tag that matches the @@ -212,6 +273,7 @@ class PageElement: criteria and appear after this Tag in the document.""" return self._findAll(name, attrs, text, limit, self.nextSiblingGenerator, **kwargs) + fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x def findPrevious(self, name=None, attrs={}, text=None, **kwargs): """Returns the first item that matches the given criteria and @@ -224,6 +286,7 @@ class PageElement: before this Tag in the document.""" return self._findAll(name, attrs, text, limit, self.previousGenerator, **kwargs) + fetchPrevious = findAllPrevious # Compatibility with pre-3.x def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs): """Returns the closest sibling to this Tag that matches the @@ -237,6 +300,7 @@ class PageElement: criteria and appear before this Tag in the document.""" return self._findAll(name, attrs, text, limit, self.previousSiblingGenerator, **kwargs) + fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x def findParent(self, name=None, attrs={}, **kwargs): """Returns the closest parent of this Tag that matches the given @@ -255,6 +319,7 @@ class PageElement: return self._findAll(name, attrs, None, limit, self.parentGenerator, **kwargs) + fetchParents = findParents # Compatibility with pre-3.x #These methods do the real heavy lifting. @@ -264,14 +329,27 @@ class PageElement: if l: r = l[0] return r - + def _findAll(self, name, attrs, text, limit, generator, **kwargs): "Iterates over a generator looking for things that match." if isinstance(name, SoupStrainer): strainer = name + # (Possibly) special case some findAll*(...) searches + elif text is None and not limit and not attrs and not kwargs: + # findAll*(True) + if name is True: + return [element for element in generator() + if isinstance(element, Tag)] + # findAll*('tag-name') + elif isinstance(name, basestring): + return [element for element in generator() + if isinstance(element, Tag) and + element.name == name] + else: + strainer = SoupStrainer(name, attrs, text, **kwargs) + # Build a SoupStrainer else: - # Build a SoupStrainer strainer = SoupStrainer(name, attrs, text, **kwargs) results = ResultSet(strainer) g = generator() @@ -289,41 +367,41 @@ class PageElement: return results #These Generators can be used to navigate starting from both - #NavigableStrings and Tags. + #NavigableStrings and Tags. def nextGenerator(self): i = self - while i: + while i is not None: i = i.next yield i def nextSiblingGenerator(self): i = self - while i: + while i is not None: i = i.nextSibling yield i def previousGenerator(self): i = self - while i: + while i is not None: i = i.previous yield i def previousSiblingGenerator(self): i = self - while i: + while i is not None: i = i.previousSibling yield i def parentGenerator(self): i = self - while i: + while i is not None: i = i.parent yield i # Utility methods def substituteEncoding(self, str, encoding=None): encoding = encoding or "utf-8" - return str.replace("%SOUP-ENCODING%", encoding) + return str.replace("%SOUP-ENCODING%", encoding) def toEncoding(self, s, encoding=None): """Encodes an object to a string in some encoding, or to Unicode. @@ -345,6 +423,21 @@ class PageElement: class NavigableString(unicode, PageElement): + def __new__(cls, value): + """Create a new NavigableString. + + When unpickling a NavigableString, this method is called with + the string in DEFAULT_OUTPUT_ENCODING. That encoding needs to be + passed in to the superclass's __new__ or the superclass won't know + how to handle non-ASCII characters. + """ + if isinstance(value, unicode): + return unicode.__new__(cls, value) + return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING) + + def __getnewargs__(self): + return (NavigableString.__str__(self),) + def __getattr__(self, attr): """text.string gives you text. This is for backwards compatibility for Navigable*String, but for CData* it lets you @@ -355,14 +448,14 @@ class NavigableString(unicode, PageElement): raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr) def __unicode__(self): - return __str__(self, None) + return str(self).decode(DEFAULT_OUTPUT_ENCODING) def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): if encoding: return self.encode(encoding) else: return self - + class CData(NavigableString): def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): @@ -377,16 +470,56 @@ class ProcessingInstruction(NavigableString): class Comment(NavigableString): def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) + return "" % NavigableString.__str__(self, encoding) class Declaration(NavigableString): def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): - return "" % NavigableString.__str__(self, encoding) + return "" % NavigableString.__str__(self, encoding) class Tag(PageElement): """Represents a found HTML tag with its attributes and contents.""" + def _invert(h): + "Cheap function to invert a hash." + i = {} + for k,v in h.items(): + i[v] = k + return i + + XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'", + "quot" : '"', + "amp" : "&", + "lt" : "<", + "gt" : ">" } + + XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS) + + def _convertEntities(self, match): + """Used in a call to re.sub to replace HTML, XML, and numeric + entities with the appropriate Unicode characters. If HTML + entities are being converted, any unrecognized entities are + escaped.""" + x = match.group(1) + if self.convertHTMLEntities and x in name2codepoint: + return unichr(name2codepoint[x]) + elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS: + if self.convertXMLEntities: + return self.XML_ENTITIES_TO_SPECIAL_CHARS[x] + else: + return u'&%s;' % x + elif len(x) > 0 and x[0] == '#': + # Handle numeric entities + if len(x) > 1 and x[1] == 'x': + return unichr(int(x[2:], 16)) + else: + return unichr(int(x[1:])) + + elif self.escapeUnrecognizedEntities: + return u'&%s;' % x + else: + return u'&%s;' % x + def __init__(self, parser, name, attrs=None, parent=None, previous=None): "Basic constructor." @@ -396,19 +529,66 @@ class Tag(PageElement): self.parserClass = parser.__class__ self.isSelfClosing = parser.isSelfClosingTag(name) self.name = name - if attrs == None: + if attrs is None: attrs = [] self.attrs = attrs self.contents = [] self.setup(parent, previous) self.hidden = False self.containsSubstitutions = False + self.convertHTMLEntities = parser.convertHTMLEntities + self.convertXMLEntities = parser.convertXMLEntities + self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities + + # Convert any HTML, XML, or numeric entities in the attribute values. + convert = lambda(k, val): (k, + re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);", + self._convertEntities, + val)) + self.attrs = map(convert, self.attrs) + + def getString(self): + if (len(self.contents) == 1 + and isinstance(self.contents[0], NavigableString)): + return self.contents[0] + + def setString(self, string): + """Replace the contents of the tag with a string""" + self.clear() + self.append(string) + + string = property(getString, setString) + + def getText(self, separator=u""): + if not len(self.contents): + return u"" + stopNode = self._lastRecursiveChild().next + strings = [] + current = self.contents[0] + while current is not stopNode: + if isinstance(current, NavigableString): + strings.append(current.strip()) + current = current.next + return separator.join(strings) + + text = property(getText) def get(self, key, default=None): """Returns the value of the 'key' attribute for the tag, or the value given for 'default' if it doesn't have that attribute.""" - return self._getAttrMap().get(key, default) + return self._getAttrMap().get(key, default) + + def clear(self): + """Extract all children.""" + for child in self.contents[:]: + child.extract() + + def index(self, element): + for i, child in enumerate(self.contents): + if child is element: + return i + raise ValueError("Tag.index: element not in tag") def has_key(self, key): return self._getAttrMap().has_key(key) @@ -433,7 +613,7 @@ class Tag(PageElement): "A tag is non-None even if it has no contents." return True - def __setitem__(self, key, value): + def __setitem__(self, key, value): """Setting tag[key] sets the value of the 'key' attribute for the tag.""" self._getAttrMap() @@ -470,6 +650,7 @@ class Tag(PageElement): return self.find(tag[:-3]) elif tag.find('__') != 0: return self.find(tag) + raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag) def __eq__(self, other): """Returns true iff this tag has the same name, the same attributes, @@ -477,6 +658,8 @@ class Tag(PageElement): NOTE: right now this will return false if two tags have the same attributes in a different order. Should this be fixed?""" + if other is self: + return True if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other): return False for i in range(0, len(self.contents)): @@ -496,6 +679,15 @@ class Tag(PageElement): def __unicode__(self): return self.__str__(None) + BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" + + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" + + ")") + + def _sub_entity(self, x): + """Used with a regular expression to substitute the + appropriate XML entity for an XML special character.""" + return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";" + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING, prettyPrint=False, indentLevel=0): """Returns a string or Unicode representation of this tag and @@ -506,20 +698,43 @@ class Tag(PageElement): the original string.""" encodedName = self.toEncoding(self.name, encoding) - + attrs = [] if self.attrs: for key, val in self.attrs: fmt = '%s="%s"' - if isString(val): + if isinstance(val, basestring): if self.containsSubstitutions and '%SOUP-ENCODING%' in val: val = self.substituteEncoding(val, encoding) + + # The attribute value either: + # + # * Contains no embedded double quotes or single quotes. + # No problem: we enclose it in double quotes. + # * Contains embedded single quotes. No problem: + # double quotes work here too. + # * Contains embedded double quotes. No problem: + # we enclose it in single quotes. + # * Embeds both single _and_ double quotes. This + # can't happen naturally, but it can happen if + # you modify an attribute value after parsing + # the document. Now we have a bit of a + # problem. We solve it by enclosing the + # attribute in single quotes, and escaping any + # embedded single quotes to XML entities. if '"' in val: fmt = "%s='%s'" - # This can't happen naturally, but it can happen - # if you modify an attribute value and print it out. if "'" in val: + # TODO: replace with apos when + # appropriate. val = val.replace("'", "&squot;") + + # Now we're okay w/r/t quotes. But the attribute + # value might also contain angle brackets, or + # ampersands that aren't part of entities. We need + # to escape those to XML entities too. + val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val) + attrs.append(fmt % (self.toEncoding(key, encoding), self.toEncoding(val, encoding))) close = '' @@ -541,7 +756,7 @@ class Tag(PageElement): s = [] attributeString = '' if attrs: - attributeString = ' ' + ' '.join(attrs) + attributeString = ' ' + ' '.join(attrs) if prettyPrint: s.append(space) s.append('<%s%s%s>' % (encodedName, attributeString, close)) @@ -558,6 +773,23 @@ class Tag(PageElement): s = ''.join(s) return s + def decompose(self): + """Recursively destroys the contents of this tree.""" + self.extract() + if len(self.contents) == 0: + return + current = self.contents[0] + while current is not None: + next = current.next + if isinstance(current, Tag): + del current.contents[:] + current.parent = None + current.previous = None + current.previousSibling = None + current.next = None + current.nextSibling = None + current = next + def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING): return self.__str__(encoding, True) @@ -573,23 +805,23 @@ class Tag(PageElement): elif isinstance(c, Tag): s.append(c.__str__(encoding, prettyPrint, indentLevel)) if text and prettyPrint: - text = text.strip() + text = text.strip() if text: if prettyPrint: s.append(" " * (indentLevel-1)) s.append(text) if prettyPrint: s.append("\n") - return ''.join(s) + return ''.join(s) #Soup methods def find(self, name=None, attrs={}, recursive=True, text=None, - **kwargs): - """Return only the first child of this - Tag matching the given criteria.""" + **kwargs): + """Return only the first child of this Tag matching the given + criteria.""" r = None - l = self.findAll(name, attrs, recursive, text, 1) + l = self.findAll(name, attrs, recursive, text, 1, **kwargs) if l: r = l[0] return r @@ -610,13 +842,17 @@ class Tag(PageElement): if not recursive: generator = self.childGenerator return self._findAll(name, attrs, text, limit, generator, **kwargs) - findAllChildren = findAll - - #Utility methods + findChildren = findAll - def append(self, tag): - """Appends the given tag to the contents of this tag.""" - self.contents.append(tag) + # Pre-3.x compatibility methods + first = find + fetch = findAll + + def fetchText(self, text=None, recursive=True, limit=None): + return self.findAll(text=text, recursive=recursive, limit=limit) + + def firstText(self, text=None, recursive=True): + return self.find(text=text, recursive=recursive) #Private methods @@ -626,29 +862,23 @@ class Tag(PageElement): if not getattr(self, 'attrMap'): self.attrMap = {} for (key, value) in self.attrs: - self.attrMap[key] = value + self.attrMap[key] = value return self.attrMap #Generator methods def childGenerator(self): - for i in range(0, len(self.contents)): - yield self.contents[i] - raise StopIteration - + # Just use the iterator from the contents + return iter(self.contents) + def recursiveChildGenerator(self): - stack = [(self, 0)] - while stack: - tag, start = stack.pop() - if isinstance(tag, Tag): - for i in range(start, len(tag.contents)): - a = tag.contents[i] - yield a - if isinstance(a, Tag) and tag.contents: - if i < len(tag.contents) - 1: - stack.append((tag, i+1)) - stack.append((a, 0)) - break - raise StopIteration + if not len(self.contents): + raise StopIteration + stopNode = self._lastRecursiveChild().next + current = self.contents[0] + while current is not stopNode: + yield current + current = current.next + # Next, a couple classes to represent queries and their results. class SoupStrainer: @@ -656,9 +886,17 @@ class SoupStrainer: text).""" def __init__(self, name=None, attrs={}, text=None, **kwargs): - self.name=name - self.attrs=attrs.copy() - self.attrs.update(kwargs) + self.name = name + if isinstance(attrs, basestring): + kwargs['class'] = _match_css_class(attrs) + attrs = None + if kwargs: + if attrs: + attrs = attrs.copy() + attrs.update(kwargs) + else: + attrs = kwargs + self.attrs = attrs self.text = text def __str__(self): @@ -666,7 +904,7 @@ class SoupStrainer: return self.text else: return "%s|%s" % (self.name, self.attrs) - + def searchTag(self, markupName=None, markupAttrs={}): found = None markup = None @@ -683,7 +921,7 @@ class SoupStrainer: if callFunctionWithTagData: match = self.name(markupName, markupAttrs) else: - match = True + match = True markupAttrMap = None for attr, matchAgainst in self.attrs.items(): if not markupAttrMap: @@ -708,8 +946,9 @@ class SoupStrainer: #print 'looking for %s in %s' % (self, markup) found = None # If given a list of items, scan it for a text element that - # matches. - if isList(markup) and not isinstance(markup, Tag): + # matches. + if hasattr(markup, "__iter__") \ + and not isinstance(markup, Tag): for element in markup: if isinstance(element, NavigableString) \ and self.search(element): @@ -722,19 +961,19 @@ class SoupStrainer: found = self.searchTag(markup) # If it's text, make sure the text matches. elif isinstance(markup, NavigableString) or \ - isString(markup): + isinstance(markup, basestring): if self._matches(markup, self.text): found = markup else: raise Exception, "I don't know how to match against a %s" \ % markup.__class__ return found - - def _matches(self, markup, matchAgainst): + + def _matches(self, markup, matchAgainst): #print "Matching %s against %s" % (markup, matchAgainst) result = False - if matchAgainst == True and type(matchAgainst) == types.BooleanType: - result = markup != None + if matchAgainst is True: + result = markup is not None elif callable(matchAgainst): result = matchAgainst(markup) else: @@ -742,17 +981,17 @@ class SoupStrainer: #other ways of matching match the tag name as a string. if isinstance(markup, Tag): markup = markup.name - if markup and not isString(markup): + if markup and not isinstance(markup, basestring): markup = unicode(markup) #Now we know that chunk is either a string, or None. if hasattr(matchAgainst, 'match'): # It's a regexp object. result = markup and matchAgainst.search(markup) - elif isList(matchAgainst): + elif hasattr(matchAgainst, '__iter__'): # list-like result = markup in matchAgainst elif hasattr(matchAgainst, 'items'): result = markup.has_key(matchAgainst) - elif matchAgainst and isString(markup): + elif matchAgainst and isinstance(markup, basestring): if isinstance(markup, unicode): matchAgainst = unicode(matchAgainst) else: @@ -771,20 +1010,6 @@ class ResultSet(list): # Now, some helper functions. -def isList(l): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is listlike.""" - return hasattr(l, '__iter__') \ - or (type(l) in (types.ListType, types.TupleType)) - -def isString(s): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is stringlike.""" - try: - return isinstance(s, unicode) or isintance(s, basestring) - except NameError: - return isinstance(s, str) - def buildTagMap(default, *args): """Turns a list of maps, lists, or scalars into a single map. Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and @@ -795,7 +1020,7 @@ def buildTagMap(default, *args): #It's a map. Merge it. for k,v in portion.items(): built[k] = v - elif isList(portion): + elif hasattr(portion, '__iter__'): # is a list #It's a list. Map each item to the default. for k in portion: built[k] = default @@ -811,7 +1036,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): """This class contains the basic parser and search code. It defines a parser that knows nothing about tag behavior except for the following: - + You can't close a tag without closing all the tags it encloses. That is, "" actually means "". @@ -824,14 +1049,11 @@ class BeautifulStoneSoup(Tag, SGMLParser): or when BeautifulSoup makes an assumption counter to what you were expecting.""" - XML_ENTITY_LIST = {} - for i in ["quot", "apos", "amp", "lt", "gt"]: - XML_ENTITY_LIST[i] = True - SELF_CLOSING_TAGS = {} NESTABLE_TAGS = {} RESET_NESTING_TAGS = {} QUOTE_TAGS = {} + PRESERVE_WHITESPACE_TAGS = [] MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'), lambda x: x.group(1) + ' />'), @@ -843,13 +1065,22 @@ class BeautifulStoneSoup(Tag, SGMLParser): HTML_ENTITIES = "html" XML_ENTITIES = "xml" + XHTML_ENTITIES = "xhtml" + # TODO: This only exists for backwards-compatibility + ALL_ENTITIES = XHTML_ENTITIES + + # Used when determining whether a text node is all whitespace and + # can be replaced with a single space. A text node that contains + # fancy Unicode spaces (usually non-breaking) should be left + # alone. + STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, } def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None, markupMassage=True, smartQuotesTo=XML_ENTITIES, - convertEntities=None, selfClosingTags=None): + convertEntities=None, selfClosingTags=None, isHTML=False): """The Soup object is initialized as the 'root tag', and the provided markup (which can be a string or a file-like object) - is fed into the underlying parser. + is fed into the underlying parser. sgmllib will process most bad HTML, and the BeautifulSoup class has some tricks for dealing with some HTML that kills @@ -876,25 +1107,54 @@ class BeautifulStoneSoup(Tag, SGMLParser): self.fromEncoding = fromEncoding self.smartQuotesTo = smartQuotesTo self.convertEntities = convertEntities + # Set the rules for how we'll deal with the entities we + # encounter if self.convertEntities: # It doesn't make sense to convert encoded characters to # entities even while you're converting entities to Unicode. # Just convert it all to Unicode. self.smartQuotesTo = None + if convertEntities == self.HTML_ENTITIES: + self.convertXMLEntities = False + self.convertHTMLEntities = True + self.escapeUnrecognizedEntities = True + elif convertEntities == self.XHTML_ENTITIES: + self.convertXMLEntities = True + self.convertHTMLEntities = True + self.escapeUnrecognizedEntities = False + elif convertEntities == self.XML_ENTITIES: + self.convertXMLEntities = True + self.convertHTMLEntities = False + self.escapeUnrecognizedEntities = False + else: + self.convertXMLEntities = False + self.convertHTMLEntities = False + self.escapeUnrecognizedEntities = False + self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags) SGMLParser.__init__(self) - + if hasattr(markup, 'read'): # It's a file-type object. markup = markup.read() self.markup = markup self.markupMassage = markupMassage try: - self._feed() + self._feed(isHTML=isHTML) except StopParsing: pass self.markup = None # The markup can now be GCed - - def _feed(self, inDocumentEncoding=None): + + def convert_charref(self, name): + """This method fixes a bug in Python's SGMLParser.""" + try: + n = int(name) + except ValueError: + return + if not 0 <= n <= 127 : # ASCII ends at 127, not 255 + return + return self.convert_codepoint(n) + + def _feed(self, inDocumentEncoding=None, isHTML=False): # Convert the document to Unicode. markup = self.markup if isinstance(markup, unicode): @@ -903,15 +1163,22 @@ class BeautifulStoneSoup(Tag, SGMLParser): else: dammit = UnicodeDammit\ (markup, [self.fromEncoding, inDocumentEncoding], - smartQuotesTo=self.smartQuotesTo) + smartQuotesTo=self.smartQuotesTo, isHTML=isHTML) markup = dammit.unicode self.originalEncoding = dammit.originalEncoding + self.declaredHTMLEncoding = dammit.declaredHTMLEncoding if markup: if self.markupMassage: - if not isList(self.markupMassage): - self.markupMassage = self.MARKUP_MASSAGE + if not hasattr(self.markupMassage, "__iter__"): + self.markupMassage = self.MARKUP_MASSAGE for fix, m in self.markupMassage: markup = fix.sub(m, markup) + # TODO: We get rid of markupMassage so that the + # soup object can be deepcopied later on. Some + # Python installations can't copy regexes. If anyone + # was relying on the existence of markupMassage, this + # might cause problems. + del(self.markupMassage) self.reset() SGMLParser.feed(self, markup) @@ -925,10 +1192,10 @@ class BeautifulStoneSoup(Tag, SGMLParser): superclass or the Tag superclass, depending on the method name.""" #print "__getattr__ called on %s.%s" % (self.__class__, methodName) - if methodName.find('start_') == 0 or methodName.find('end_') == 0 \ - or methodName.find('do_') == 0: + if methodName.startswith('start_') or methodName.startswith('end_') \ + or methodName.startswith('do_'): return SGMLParser.__getattr__(self, methodName) - elif methodName.find('__') != 0: + elif not methodName.startswith('__'): return Tag.__getattr__(self, methodName) else: raise AttributeError @@ -938,7 +1205,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): self-closing tag according to this parser.""" return self.SELF_CLOSING_TAGS.has_key(name) \ or self.instanceSelfClosingTags.has_key(name) - + def reset(self): Tag.__init__(self, self, self.ROOT_TAG_NAME) self.hidden = 1 @@ -948,15 +1215,9 @@ class BeautifulStoneSoup(Tag, SGMLParser): self.tagStack = [] self.quoteStack = [] self.pushTag(self) - + def popTag(self): tag = self.tagStack.pop() - # Tags with just one string-owning child get the child as a - # 'string' property, so that soup.tag.string is shorthand for - # soup.tag.contents[0] - if len(self.currentTag.contents) == 1 and \ - isinstance(self.currentTag.contents[0], NavigableString): - self.currentTag.string = self.currentTag.contents[0] #print "Pop", tag.name if self.tagStack: @@ -966,14 +1227,16 @@ class BeautifulStoneSoup(Tag, SGMLParser): def pushTag(self, tag): #print "Push", tag.name if self.currentTag: - self.currentTag.append(tag) + self.currentTag.contents.append(tag) self.tagStack.append(tag) self.currentTag = self.tagStack[-1] def endData(self, containerClass=NavigableString): if self.currentData: - currentData = ''.join(self.currentData) - if not currentData.strip(): + currentData = u''.join(self.currentData) + if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and + not set([tag.name for tag in self.tagStack]).intersection( + self.PRESERVE_WHITESPACE_TAGS)): if '\n' in currentData: currentData = '\n' else: @@ -998,7 +1261,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): the given tag.""" #print "Popping to %s" % name if name == self.ROOT_TAG_NAME: - return + return numPops = 0 mostRecentTag = None @@ -1011,7 +1274,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): for i in range(0, numPops): mostRecentTag = self.popTag() - return mostRecentTag + return mostRecentTag def _smartPop(self, name): @@ -1022,10 +1285,9 @@ class BeautifulStoneSoup(Tag, SGMLParser): comes between this tag and the previous tag of this type. Examples: -

FooBar

should pop to 'p', not 'b'. -

FooBar

should pop to 'table', not 'p'. -

Foo

Bar

should pop to 'tr', not 'p'. -

FooBar

should pop to 'p', not 'b'. +

FooBar *

* should pop to 'p', not 'b'. +

Foo

Bar *

* should pop to 'table', not 'p'. +

Foo

Bar *

* should pop to 'tr', not 'p'.

    • *
    • * should pop to 'ul', not the first 'li'.
    • ** should pop to 'table', not the first 'tr' @@ -1044,11 +1306,11 @@ class BeautifulStoneSoup(Tag, SGMLParser): #last occurance. popTo = name break - if (nestingResetTriggers != None + if (nestingResetTriggers is not None and p.name in nestingResetTriggers) \ - or (nestingResetTriggers == None and isResetNesting + or (nestingResetTriggers is None and isResetNesting and self.RESET_NESTING_TAGS.has_key(p.name)): - + #If we encounter one of the nesting reset triggers #peculiar to this tag, or we encounter another tag #that causes nesting to reset, pop up to but not @@ -1061,13 +1323,13 @@ class BeautifulStoneSoup(Tag, SGMLParser): self._popToTag(popTo, inclusive) def unknown_starttag(self, name, attrs, selfClosing=0): - #print "Start tag %s" % name + #print "Start tag %s: %s" % (name, attrs) if self.quoteStack: #This is not a real tag. #print "<%s> is not real!" % name - attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs)) + attrs = ''.join([' %s="%s"' % (x, y) for x, y in attrs]) self.handle_data('<%s%s>' % (name, attrs)) - return + return self.endData() if not self.isSelfClosingTag(name) and not selfClosing: @@ -1083,7 +1345,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): self.previous = tag self.pushTag(tag) if selfClosing or self.isSelfClosingTag(name): - self.popTag() + self.popTag() if name in self.QUOTE_TAGS: #print "Beginning quote (%s)" % name self.quoteStack.append(name) @@ -1118,7 +1380,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): object, possibly one with a %SOUP-ENCODING% slot into which an encoding will be plugged later.""" if text[:3] == "xml": - text = "xml version='1.0' encoding='%SOUP-ENCODING%'" + text = u"xml version='1.0' encoding='%SOUP-ENCODING%'" self._toStringSubclass(text, ProcessingInstruction) def handle_comment(self, text): @@ -1127,8 +1389,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): def handle_charref(self, ref): "Handle character references as data." - if self.convertEntities in [self.HTML_ENTITIES, - self.XML_ENTITIES]: + if self.convertEntities: data = unichr(int(ref)) else: data = '&#%s;' % ref @@ -1136,20 +1397,47 @@ class BeautifulStoneSoup(Tag, SGMLParser): def handle_entityref(self, ref): """Handle entity references as data, possibly converting known - HTML entity references to the corresponding Unicode + HTML and/or XML entity references to the corresponding Unicode characters.""" data = None - if self.convertEntities == self.HTML_ENTITIES or \ - (self.convertEntities == self.XML_ENTITIES and \ - self.XML_ENTITY_LIST.get(ref)): + if self.convertHTMLEntities: try: data = unichr(name2codepoint[ref]) except KeyError: pass + + if not data and self.convertXMLEntities: + data = self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref) + + if not data and self.convertHTMLEntities and \ + not self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref): + # TODO: We've got a problem here. We're told this is + # an entity reference, but it's not an XML entity + # reference or an HTML entity reference. Nonetheless, + # the logical thing to do is to pass it through as an + # unrecognized entity reference. + # + # Except: when the input is "&carol;" this function + # will be called with input "carol". When the input is + # "AT&T", this function will be called with input + # "T". We have no way of knowing whether a semicolon + # was present originally, so we don't know whether + # this is an unknown entity or just a misplaced + # ampersand. + # + # The more common case is a misplaced ampersand, so I + # escape the ampersand and omit the trailing semicolon. + data = "&%s" % ref if not data: - data = '&%s;' % ref + # This case is different from the one above, because we + # haven't already gone through a supposedly comprehensive + # mapping of entities to Unicode characters. We might not + # have gone through any mapping at all. So the chances are + # very high that this is a real entity, and not a + # misplaced ampersand. + data = "&%s;" % ref self.handle_data(data) - + def handle_decl(self, data): "Handle DOCTYPEs and the like as Declaration objects." self._toStringSubclass(data, Declaration) @@ -1225,26 +1513,29 @@ class BeautifulSoup(BeautifulStoneSoup): def __init__(self, *args, **kwargs): if not kwargs.has_key('smartQuotesTo'): kwargs['smartQuotesTo'] = self.HTML_ENTITIES + kwargs['isHTML'] = True BeautifulStoneSoup.__init__(self, *args, **kwargs) SELF_CLOSING_TAGS = buildTagMap(None, - ['br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base']) + ('br' , 'hr', 'input', 'img', 'meta', + 'spacer', 'link', 'frame', 'base', 'col')) + + PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea']) + + QUOTE_TAGS = {'script' : None, 'textarea' : None} - QUOTE_TAGS = {'script': None} - #According to the HTML standard, each of these inline tags can #contain another tag of the same type. Furthermore, it's common #to actually use these tags this way. - NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', - 'center'] + NESTABLE_INLINE_TAGS = ('span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', + 'center') #According to the HTML standard, these block tags can contain #another tag of the same type. Furthermore, it's common #to actually use these tags this way. - NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del'] + NESTABLE_BLOCK_TAGS = ('blockquote', 'div', 'fieldset', 'ins', 'del') - #Lists can contain other lists, but there are restrictions. + #Lists can contain other lists, but there are restrictions. NESTABLE_LIST_TAGS = { 'ol' : [], 'ul' : [], 'li' : ['ul', 'ol'], @@ -1252,8 +1543,8 @@ class BeautifulSoup(BeautifulStoneSoup): 'dd' : ['dl'], 'dt' : ['dl'] } - #Tables can contain other tables, but there are restrictions. - NESTABLE_TABLE_TAGS = {'table' : [], + #Tables can contain other tables, but there are restrictions. + NESTABLE_TABLE_TAGS = {'table' : [], 'tr' : ['table', 'tbody', 'tfoot', 'thead'], 'td' : ['tr'], 'th' : ['tr'], @@ -1262,7 +1553,7 @@ class BeautifulSoup(BeautifulStoneSoup): 'tfoot' : ['table'], } - NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre'] + NON_NESTABLE_BLOCK_TAGS = ('address', 'form', 'p', 'pre') #If one of these tags is encountered, all tags up to the next tag of #this type are popped. @@ -1275,7 +1566,7 @@ class BeautifulSoup(BeautifulStoneSoup): NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS) # Used to detect the charset in a META tag; see start_meta - CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)") + CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)", re.M) def start_meta(self, attrs): """Beautiful Soup can detect a charset included in a META tag, @@ -1298,32 +1589,35 @@ class BeautifulSoup(BeautifulStoneSoup): if httpEquiv and contentType: # It's an interesting meta tag. match = self.CHARSET_RE.search(contentType) if match: - if getattr(self, 'declaredHTMLEncoding') or \ - (self.originalEncoding == self.fromEncoding): - # This is our second pass through the document, or - # else an encoding was specified explicitly and it - # worked. Rewrite the meta tag. - newAttr = self.CHARSET_RE.sub\ - (lambda(match):match.group(1) + - "%SOUP-ENCODING%", value) + if (self.declaredHTMLEncoding is not None or + self.originalEncoding == self.fromEncoding): + # An HTML encoding was sniffed while converting + # the document to Unicode, or an HTML encoding was + # sniffed during a previous pass through the + # document, or an encoding was specified + # explicitly and it worked. Rewrite the meta tag. + def rewrite(match): + return match.group(1) + "%SOUP-ENCODING%" + newAttr = self.CHARSET_RE.sub(rewrite, contentType) attrs[contentTypeIndex] = (attrs[contentTypeIndex][0], newAttr) tagNeedsEncodingSubstitution = True else: # This is our first pass through the document. - # Go through it again with the new information. + # Go through it again with the encoding information. newCharset = match.group(3) if newCharset and newCharset != self.originalEncoding: self.declaredHTMLEncoding = newCharset self._feed(self.declaredHTMLEncoding) raise StopParsing + pass tag = self.unknown_starttag("meta", attrs) - if tagNeedsEncodingSubstitution: + if tag and tagNeedsEncodingSubstitution: tag.containsSubstitutions = True class StopParsing(Exception): pass - + class ICantBelieveItsBeautifulSoup(BeautifulSoup): """The BeautifulSoup class is oriented towards skipping over @@ -1350,11 +1644,11 @@ class ICantBelieveItsBeautifulSoup(BeautifulSoup): wouldn't be.""" I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \ - ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', + ('em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b', - 'big'] + 'big') - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript'] + I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ('noscript',) NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS, I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS, @@ -1369,7 +1663,7 @@ class MinimalSoup(BeautifulSoup): This also makes it better for subclassing than BeautifulStoneSoup or BeautifulSoup.""" - + RESET_NESTING_TAGS = buildTagMap('noscript') NESTABLE_TAGS = {} @@ -1399,7 +1693,7 @@ class BeautifulSOAP(BeautifulStoneSoup): parent = self.tagStack[-2] parent._getAttrMap() if (isinstance(tag, Tag) and len(tag.contents) == 1 and - isinstance(tag.contents[0], NavigableString) and + isinstance(tag.contents[0], NavigableString) and not parent.attrMap.has_key(tag.name)): parent[tag.name] = tag.contents[0] BeautifulStoneSoup.popTag(self) @@ -1409,7 +1703,7 @@ class BeautifulSOAP(BeautifulStoneSoup): #and "unprofessional" for use in enterprise screen-scraping. We feel #your pain! For such-minded folk, the Beautiful Soup Consortium And #All-Night Kosher Bakery recommends renaming this file to -#"RobustParser.py" (or, in cases of extreme enterprisitude, +#"RobustParser.py" (or, in cases of extreme enterprisiness, #"RobustParserBeanInterface.class") and using the following #enterprise-friendly class aliases: class RobustXMLParser(BeautifulStoneSoup): @@ -1439,20 +1733,19 @@ try: import chardet # import chardet.constants # chardet.constants._debug = 1 -except: +except ImportError: chardet = None -chardet = None # cjkcodecs and iconv_codec make Python know about more character encodings. # Both are available from http://cjkpython.i18n.org/ # They're built in if you use Python 2.4. try: import cjkcodecs.aliases -except: +except ImportError: pass try: import iconv_codec -except: +except ImportError: pass class UnicodeDammit: @@ -1467,15 +1760,18 @@ class UnicodeDammit: # by the heuristics in find_codec. CHARSET_ALIASES = { "macintosh" : "mac-roman", "x-sjis" : "shift-jis" } - + def __init__(self, markup, overrideEncodings=[], - smartQuotesTo='xml'): + smartQuotesTo='xml', isHTML=False): + self.declaredHTMLEncoding = None self.markup, documentEncoding, sniffedEncoding = \ - self._detectEncoding(markup) + self._detectEncoding(markup, isHTML) self.smartQuotesTo = smartQuotesTo self.triedEncodings = [] - if isinstance(markup, unicode): - return markup + if markup == '' or isinstance(markup, unicode): + self.originalEncoding = None + self.unicode = unicode(markup) + return u = None for proposedEncoding in overrideEncodings: @@ -1485,7 +1781,7 @@ class UnicodeDammit: for proposedEncoding in (documentEncoding, sniffedEncoding): u = self._convertFrom(proposedEncoding) if u: break - + # If no luck and we have auto-detection library, try that: if not u and chardet and not isinstance(self.markup, unicode): u = self._convertFrom(chardet.detect(self.markup)['encoding']) @@ -1495,6 +1791,7 @@ class UnicodeDammit: for proposed_encoding in ("utf-8", "windows-1252"): u = self._convertFrom(proposed_encoding) if u: break + self.unicode = u if not u: self.originalEncoding = None @@ -1502,14 +1799,14 @@ class UnicodeDammit: """Changes a MS smart quote character to an XML or HTML entity.""" sub = self.MS_CHARS.get(orig) - if type(sub) == types.TupleType: + if isinstance(sub, tuple): if self.smartQuotesTo == 'xml': sub = '&#x%s;' % sub[1] else: sub = '&%s;' % sub[0] - return sub + return sub - def _convertFrom(self, proposed): + def _convertFrom(self, proposed): proposed = self.find_codec(proposed) if not proposed or proposed in self.triedEncodings: return None @@ -1518,9 +1815,9 @@ class UnicodeDammit: # Convert smart quotes to HTML if coming from an encoding # that might have them. - if self.smartQuotesTo and proposed in("windows-1252", - "ISO-8859-1", - "ISO-8859-2"): + if self.smartQuotesTo and proposed.lower() in("windows-1252", + "iso-8859-1", + "iso-8859-2"): markup = re.compile("([\x80-\x9f])").sub \ (lambda(x): self._subMSChar(x.group(1)), markup) @@ -1528,12 +1825,12 @@ class UnicodeDammit: try: # print "Trying to convert document to %s" % proposed u = self._toUnicode(markup, proposed) - self.markup = u + self.markup = u self.originalEncoding = proposed except Exception, e: # print "That didn't work!" # print e - return None + return None #print "Correct encoding: %s" % proposed return self.markup @@ -1561,8 +1858,8 @@ class UnicodeDammit: data = data[4:] newdata = unicode(data, encoding) return newdata - - def _detectEncoding(self, xml_data): + + def _detectEncoding(self, xml_data, isHTML=False): """Given a document, tries to detect its XML encoding.""" xml_encoding = sniffed_xml_encoding = None try: @@ -1610,13 +1907,17 @@ class UnicodeDammit: else: sniffed_xml_encoding = 'ascii' pass - xml_encoding_match = re.compile \ - ('^<\?.*encoding=[\'"](.*?)[\'"].*\?>')\ - .match(xml_data) except: xml_encoding_match = None - if xml_encoding_match: + xml_encoding_match = re.compile( + '^<\?.*encoding=[\'"](.*?)[\'"].*\?>').match(xml_data) + if not xml_encoding_match and isHTML: + regexp = re.compile('<\s*meta[^>]+charset=([^>]*?)[;\'">]', re.I) + xml_encoding_match = regexp.search(xml_data) + if xml_encoding_match is not None: xml_encoding = xml_encoding_match.groups()[0].lower() + if isHTML: + self.declaredHTMLEncoding = xml_encoding if sniffed_xml_encoding and \ (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode', 'iso-10646-ucs-4', 'ucs-4', 'csucs4', @@ -1633,12 +1934,12 @@ class UnicodeDammit: or charset def _codec(self, charset): - if not charset: return charset + if not charset: return charset codec = None try: codecs.lookup(charset) codec = charset - except LookupError: + except (LookupError, ValueError): pass return codec @@ -1707,5 +2008,5 @@ class UnicodeDammit: #By default, act as an HTML pretty-printer. if __name__ == '__main__': import sys - soup = BeautifulStoneSoup(sys.stdin.read()) + soup = BeautifulSoup(sys.stdin) print soup.prettify() diff --git a/constants.py b/constants.py index 6af00eb..a869143 100644 --- a/constants.py +++ b/constants.py @@ -36,7 +36,8 @@ CONTENT_START = ''' %s %s en-UK - + + fanfiction sgzmd urn:uuid:sigizmund.com062820072147132 diff --git a/ffnet.py b/ffnet.py index 0816bf8..d576ce6 100644 --- a/ffnet.py +++ b/ffnet.py @@ -118,20 +118,39 @@ class FFNet(FanfictionSiteAdapter): textbuf = '' emit = False - for l in lines: - if l.find('') != -1: - #s2 = bs.BeautifulStoneSoup(l) - #return s2.div.prettify() - emit = True - - if emit: - textbuf = textbuf + "\n" + l - - if l.find('') != -1: - emit = False + olddata = data + try: + data = data.decode('utf8') + except: + data = olddata - s2 = bs.BeautifulStoneSoup(textbuf) - return s2.div.prettify() + try: + soup = bs.BeautifulStoneSoup(data) + except: + logging.info("Failed to decode: <%s>" % data) + soup = None + div = soup.find('div', {'id' : 'storytext'}) + if None == div: + return '' + + return div.prettify() + + # + # for l in lines: + # if l.find("
    ') != -1 or l.find('
    ") != -1 or l.find('') != -1: + # emit = False + # + # s2 = bs.BeautifulStoneSoup(textbuf) + # return s2.div.prettify() def setLogin(self, login): @@ -162,7 +181,7 @@ class FFA_UnitTests(unittest.TestCase): f = FFNet('http://www.fanfiction.net/s/5257563/1') urls = f.extractIndividualUrls() - self.assertEquals(8, len(urls)) + self.assertEquals(10, len(urls)) def testGetText(self): url = 'http://www.fanfiction.net/s/5257563/1' diff --git a/ficwad.py b/ficwad.py index 35b3b76..26be350 100644 --- a/ficwad.py +++ b/ficwad.py @@ -17,10 +17,10 @@ class FicWad(FanfictionSiteAdapter): self.url = url self.host = up.urlparse(url).netloc - def requiresLogin(self, url): + def requiresLogin(self, url = None): return False - def performLogin(self, url): + def performLogin(self, url = None): pass def setLogin(self, login): @@ -37,7 +37,7 @@ class FicWad(FanfictionSiteAdapter): self.storyName = title.split('::')[0].strip() author = soup.find('span', {'class' : 'author'}) - self.authorName = author.a.string + self.authorName = str(author.a.string) print('Story "%s" by %s' % (self.storyName, self.authorName)) diff --git a/html2text.py b/html2text.py new file mode 100644 index 0000000..5d61b9a --- /dev/null +++ b/html2text.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python +"""html2text: Turn HTML into equivalent Markdown-structured text.""" +__version__ = "2.37" +__author__ = "Aaron Swartz (me@aaronsw.com)" +__copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3." +__contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"] + +# TODO: +# Support decoded entities with unifiable. + +if not hasattr(__builtins__, 'True'): True, False = 1, 0 +import re, sys, urllib, htmlentitydefs, codecs, StringIO, types +import sgmllib +import urlparse +sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]') + +try: from textwrap import wrap +except: pass + +# Use Unicode characters instead of their ascii psuedo-replacements +UNICODE_SNOB = 0 + +# Put the links after each paragraph instead of at the end. +LINKS_EACH_PARAGRAPH = 0 + +# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.) +BODY_WIDTH = 78 + +# Don't show internal links (href="#local-anchor") -- corresponding link targets +# won't be visible in the plain text file anyway. +SKIP_INTERNAL_LINKS = False + +### Entity Nonsense ### + +def name2cp(k): + if k == 'apos': return ord("'") + if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3 + return htmlentitydefs.name2codepoint[k] + else: + k = htmlentitydefs.entitydefs[k] + if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1 + return ord(codecs.latin_1_decode(k)[0]) + +unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"', +'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*', +'ndash':'-', 'oelig':'oe', 'aelig':'ae', +'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a', +'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e', +'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i', +'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o', +'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u'} + +unifiable_n = {} + +for k in unifiable.keys(): + unifiable_n[name2cp(k)] = unifiable[k] + +def charref(name): + if name[0] in ['x','X']: + c = int(name[1:], 16) + else: + c = int(name) + + if not UNICODE_SNOB and c in unifiable_n.keys(): + return unifiable_n[c] + else: + return unichr(c) + +def entityref(c): + if not UNICODE_SNOB and c in unifiable.keys(): + return unifiable[c] + else: + try: name2cp(c) + except KeyError: return "&" + c + else: return unichr(name2cp(c)) + +def replaceEntities(s): + s = s.group(1) + if s[0] == "#": + return charref(s[1:]) + else: return entityref(s) + +r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));") +def unescape(s): + return r_unescape.sub(replaceEntities, s) + +def fixattrs(attrs): + # Fix bug in sgmllib.py + if not attrs: return attrs + newattrs = [] + for attr in attrs: + newattrs.append((attr[0], unescape(attr[1]))) + return newattrs + +### End Entity Nonsense ### + +def onlywhite(line): + """Return true if the line does only consist of whitespace characters.""" + for c in line: + if c is not ' ' and c is not ' ': + return c is ' ' + return line + +def optwrap(text): + """Wrap all paragraphs in the provided text.""" + if not BODY_WIDTH: + return text + + assert wrap, "Requires Python 2.3." + result = '' + newlines = 0 + for para in text.split("\n"): + if len(para) > 0: + if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*': + for line in wrap(para, BODY_WIDTH): + result += line + "\n" + result += "\n" + newlines = 2 + else: + if not onlywhite(para): + result += para + "\n" + newlines = 1 + else: + if newlines < 2: + result += "\n" + newlines += 1 + return result + +def hn(tag): + if tag[0] == 'h' and len(tag) == 2: + try: + n = int(tag[1]) + if n in range(1, 10): return n + except ValueError: return 0 + +class _html2text(sgmllib.SGMLParser): + def __init__(self, out=None, baseurl=''): + sgmllib.SGMLParser.__init__(self) + + if out is None: self.out = self.outtextf + else: self.out = out + self.outtext = u'' + self.quiet = 0 + self.p_p = 0 + self.outcount = 0 + self.start = 1 + self.space = 0 + self.a = [] + self.astack = [] + self.acount = 0 + self.list = [] + self.blockquote = 0 + self.pre = 0 + self.startpre = 0 + self.lastWasNL = 0 + self.abbr_title = None # current abbreviation definition + self.abbr_data = None # last inner HTML (for abbr being defined) + self.abbr_list = {} # stack of abbreviations to write later + self.baseurl = baseurl + + def outtextf(self, s): + self.outtext += s + + def close(self): + sgmllib.SGMLParser.close(self) + + self.pbr() + self.o('', 0, 'end') + + return self.outtext + + def handle_charref(self, c): + self.o(charref(c)) + + def handle_entityref(self, c): + self.o(entityref(c)) + + def unknown_starttag(self, tag, attrs): + self.handle_tag(tag, attrs, 1) + + def unknown_endtag(self, tag): + self.handle_tag(tag, None, 0) + + def previousIndex(self, attrs): + """ returns the index of certain set of attributes (of a link) in the + self.a list + + If the set of attributes is not found, returns None + """ + if not attrs.has_key('href'): return None + + i = -1 + for a in self.a: + i += 1 + match = 0 + + if a.has_key('href') and a['href'] == attrs['href']: + if a.has_key('title') or attrs.has_key('title'): + if (a.has_key('title') and attrs.has_key('title') and + a['title'] == attrs['title']): + match = True + else: + match = True + + if match: return i + + def handle_tag(self, tag, attrs, start): + attrs = fixattrs(attrs) + + if hn(tag): + self.p() + if start: self.o(hn(tag)*"#" + ' ') + + if tag in ['p', 'div']: self.p() + + if tag == "br" and start: self.o(" \n") + + if tag == "hr" and start: + self.p() + self.o("* * *") + self.p() + + if tag in ["head", "style", 'script']: + if start: self.quiet += 1 + else: self.quiet -= 1 + + if tag in ["body"]: + self.quiet = 0 # sites like 9rules.com never close + + if tag == "blockquote": + if start: + self.p(); self.o('> ', 0, 1); self.start = 1 + self.blockquote += 1 + else: + self.blockquote -= 1 + self.p() + + if tag in ['em', 'i', 'u']: self.o("_") + if tag in ['strong', 'b']: self.o("**") + if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` `` + if tag == "abbr": + if start: + attrsD = {} + for (x, y) in attrs: attrsD[x] = y + attrs = attrsD + + self.abbr_title = None + self.abbr_data = '' + if attrs.has_key('title'): + self.abbr_title = attrs['title'] + else: + if self.abbr_title != None: + self.abbr_list[self.abbr_data] = self.abbr_title + self.abbr_title = None + self.abbr_data = '' + + if tag == "a": + if start: + attrsD = {} + for (x, y) in attrs: attrsD[x] = y + attrs = attrsD + if attrs.has_key('href') and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')): + self.astack.append(attrs) + self.o("[") + else: + self.astack.append(None) + else: + if self.astack: + a = self.astack.pop() + if a: + i = self.previousIndex(a) + if i is not None: + a = self.a[i] + else: + self.acount += 1 + a['count'] = self.acount + a['outcount'] = self.outcount + self.a.append(a) + self.o("][" + `a['count']` + "]") + + if tag == "img" and start: + attrsD = {} + for (x, y) in attrs: attrsD[x] = y + attrs = attrsD + if attrs.has_key('src'): + attrs['href'] = attrs['src'] + alt = attrs.get('alt', '') + i = self.previousIndex(attrs) + if i is not None: + attrs = self.a[i] + else: + self.acount += 1 + attrs['count'] = self.acount + attrs['outcount'] = self.outcount + self.a.append(attrs) + self.o("![") + self.o(alt) + self.o("]["+`attrs['count']`+"]") + + if tag == 'dl' and start: self.p() + if tag == 'dt' and not start: self.pbr() + if tag == 'dd' and start: self.o(' ') + if tag == 'dd' and not start: self.pbr() + + if tag in ["ol", "ul"]: + if start: + self.list.append({'name':tag, 'num':0}) + else: + if self.list: self.list.pop() + + self.p() + + if tag == 'li': + if start: + self.pbr() + if self.list: li = self.list[-1] + else: li = {'name':'ul', 'num':0} + self.o(" "*len(self.list)) #TODO: line up
    1. s > 9 correctly. + if li['name'] == "ul": self.o("* ") + elif li['name'] == "ol": + li['num'] += 1 + self.o(`li['num']`+". ") + self.start = 1 + else: + self.pbr() + + if tag in ["table", "tr"] and start: self.p() + if tag == 'td': self.pbr() + + if tag == "pre": + if start: + self.startpre = 1 + self.pre = 1 + else: + self.pre = 0 + self.p() + + def pbr(self): + if self.p_p == 0: self.p_p = 1 + + def p(self): self.p_p = 2 + + def o(self, data, puredata=0, force=0): + if self.abbr_data is not None: self.abbr_data += data + + if not self.quiet: + if puredata and not self.pre: + data = re.sub('\s+', ' ', data) + if data and data[0] == ' ': + self.space = 1 + data = data[1:] + if not data and not force: return + + if self.startpre: + #self.out(" :") #TODO: not output when already one there + self.startpre = 0 + + bq = (">" * self.blockquote) + if not (force and data and data[0] == ">") and self.blockquote: bq += " " + + if self.pre: + bq += " " + data = data.replace("\n", "\n"+bq) + + if self.start: + self.space = 0 + self.p_p = 0 + self.start = 0 + + if force == 'end': + # It's the end. + self.p_p = 0 + self.out("\n") + self.space = 0 + + + if self.p_p: + self.out(('\n'+bq)*self.p_p) + self.space = 0 + + if self.space: + if not self.lastWasNL: self.out(' ') + self.space = 0 + + if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"): + if force == "end": self.out("\n") + + newa = [] + for link in self.a: + if self.outcount > link['outcount']: + self.out(" ["+`link['count']`+"]: " + urlparse.urljoin(self.baseurl, link['href'])) + if link.has_key('title'): self.out(" ("+link['title']+")") + self.out("\n") + else: + newa.append(link) + + if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done. + + self.a = newa + + if self.abbr_list and force == "end": + for abbr, definition in self.abbr_list.items(): + self.out(" *[" + abbr + "]: " + definition + "\n") + + self.p_p = 0 + self.out(data) + self.lastWasNL = data and data[-1] == '\n' + self.outcount += 1 + + def handle_data(self, data): + if r'\/script>' in data: self.quiet -= 1 + self.o(data, 1) + + def unknown_decl(self, data): pass + +def wrapwrite(text): sys.stdout.write(text.encode('utf8')) + +def html2text_file(html, out=wrapwrite, baseurl=''): + h = _html2text(out, baseurl) + h.feed(html) + h.feed("") + return h.close() + +def html2text(html, baseurl=''): + return optwrap(html2text_file(html, None, baseurl)) + +if __name__ == "__main__": + baseurl = '' + if sys.argv[1:]: + arg = sys.argv[1] + if arg.startswith('http://'): + baseurl = arg + j = urllib.urlopen(baseurl) + try: + from feedparser import _getCharacterEncoding as enc + except ImportError: + enc = lambda x, y: ('utf-8', 1) + text = j.read() + encoding = enc(j.headers, text)[0] + if encoding == 'us-ascii': encoding = 'utf-8' + data = text.decode(encoding) + + else: + encoding = 'utf8' + if len(sys.argv) > 2: + encoding = sys.argv[2] + data = open(arg, 'r').read().decode(encoding) + else: + data = sys.stdin.read().decode('utf8') + wrapwrite(html2text(data, baseurl)) diff --git a/output.py b/output.py index aeea8cd..79870c0 100644 --- a/output.py +++ b/output.py @@ -25,6 +25,7 @@ import zipdir import html_constants from constants import * +import html2text class FanficWriter: @@ -37,6 +38,22 @@ class FanficWriter: def finalise(self): pass +class TextWriter(FanficWriter): + htmlWriter = None + + def __init__(self, base, name, author, inmemory=False, compress=False): + self.htmlWriter = HTMLWriter(base, name, author, True, False) + + def writeChapter(self, title, text): + self.htmlWriter.writeChapter(title, text) + + def finalise(self): + self.htmlWriter.finalise() + self.output = StringIO.StringIO() + self.output.write(html2text.html2text(self.htmlWriter.output.getvalue().decode('utf-8')).encode('utf-8')) + self.name = self.htmlWriter.name + + class HTMLWriter(FanficWriter): body = '' @@ -59,10 +76,17 @@ class HTMLWriter(FanficWriter): self.xhtmlTemplate = string.Template(html_constants.XHTML_START) self.chapterStartTemplate = string.Template(html_constants.XHTML_CHAPTER_START) - + + def _printableVersion(self, text): + try: + d = text.decode('utf-8') + return d + except: + return text + def writeChapter(self, title, text): - title = title.decode('utf-8') - text = text.decode('utf-8') + title = self._printableVersion(title) #title.decode('utf-8') + text = self._printableVersion(text) #text.decode('utf-8') self.body = self.body + '\n' + self.chapterStartTemplate.substitute({'chapter' : title}) self.body = self.body + '\n' + text diff --git a/twilighted.py b/twilighted.py new file mode 100644 index 0000000..54ff35d --- /dev/null +++ b/twilighted.py @@ -0,0 +1,134 @@ +import os +import re +import sys +import shutil +import os.path +import urllib as u +import logging +import pprint as pp +import unittest +import urllib2 as u2 +import urlparse as up +import BeautifulSoup as bs +import htmlentitydefs as hdefs + +from adapter import * +import twipassword + +class Twilighted(FanfictionSiteAdapter): + def __init__(self, url): + self.url = url + parsedUrl = up.urlparse(url) + self.host = parsedUrl.netloc + self.path = parsedUrl.path + self.opener = u2.build_opener(u2.HTTPCookieProcessor()) + self.password=twipassword.password + self.login='sigizmund' + logging.debug("Created Twilighted: url=%s" % (self.url)) + + + def requiresLogin(self, url = None): + return True + + def performLogin(self, url = None): + data = {} + + data['penname'] = self.login + data['password'] = self.password + data['cookiecheck'] = '1' + data['submit'] = 'Submit' + + urlvals = u.urlencode(data) + loginUrl = 'http://' + self.host + self._getLoginScript() + logging.debug("Will now login to URL %s" % loginUrl) + + req = self.opener.open(loginUrl, urlvals) + + d = req.read().decode('utf-8') + + if self.reqLoginData(d) : + return False + else: + return True + + + def setLogin(self, login): + self.login = login + + def setPassword(self, password): + self.password = password + + def extractIndividualUrls(self): + data = self.opener.open(self.url).read() + + if self.reqLoginData(data): + self.performLogin() + data = self.opener.open(self.url).read() + if self.reqLoginData(data): + return None + + soup = bs.BeautifulStoneSoup(data) + + title = soup.find('title').string + self.storyName = title.split(' by ')[0].strip() + self.authorName = title.split(' by ')[1].strip() + + select = soup.find('select', { 'name' : 'chapter' } ) + + allOptions = select.findAll('option') + result = [] + for o in allOptions: + url = self.url + "&chapter=%s" % o['value'] + title = o.string + result.append((url,title)) + + return result + + def getStoryName(self): + return self.storyName + + def getAuthorName(self): + return self.authorName + + def getText(self, url): + if url.find('http://') == -1: + url = 'http://' + self.host + '/' + url + + logging.debug('Getting data from: %s' % url) + + data = self.opener.open(url).read() + + soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) + + div = soup.find('div', {'id' : 'story'}) + + if None == div: + return '' + + return div.prettify() + + def _getLoginScript(self): + return '/user.php?action=login' + + def reqLoginData(self, data): + if data.find('Registered Users Only. Please click OK to login or register.') != -1 or data.find('There is no such account on our website') != -1: + return True + else: + return False + + +class Twilighted_UnitTests(unittest.TestCase): + def setUp(self): + logging.basicConfig(level=logging.DEBUG) + pass + + def testLoginWorks(self): + url = 'http://www.twilighted.net/viewstory.php?sid=10004' + self.assertTrue(Twilighted(url).performLogin()) + + def testGetUrlsWorks(self): + url = 'http://www.twilighted.net/viewstory.php?sid=10004' + self.assertEquals(32, len(Twilighted(url).extractIndividualUrls())) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file