1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
|
"""Utilities for working with HTML."""
import html5lib
from html5lib import sanitizer, serializer, tokenizer, treebuilders, treewalkers
import re, htmlentitydefs
class HTMLSanitizerMixin(sanitizer.HTMLSanitizerMixin):
acceptable_elements = ('a', 'abbr', 'acronym', 'address', 'b', 'big',
'blockquote', 'br', 'caption', 'center', 'cite', 'code', 'col',
'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'font',
'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'ins', 'kbd',
'li', 'ol', 'p', 'pre', 'q', 's', 'samp', 'small', 'span', 'strike',
'strong', 'sub', 'sup', 'table', 'tbody', 'td', 'tfoot', 'th', 'thead',
'tr', 'tt', 'u', 'ul', 'var', 'object', 'param')
acceptable_attributes = ('abbr', 'align', 'alt', 'axis', 'border',
'cellpadding', 'cellspacing', 'char', 'charoff', 'charset', 'cite',
'cols', 'colspan', 'data', 'datetime', 'dir', 'frame', 'headers', 'height',
'href', 'hreflang', 'hspace', 'lang', 'longdesc', 'name', 'nohref',
'noshade', 'nowrap', 'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope',
'span', 'src', 'start', 'summary', 'title', 'type', 'valign', 'vspace',
'width')
allowed_elements = acceptable_elements
allowed_attributes = acceptable_attributes
allowed_css_properties = ()
allowed_css_keywords = ()
allowed_svg_properties = ()
class HTMLSanitizer(tokenizer.HTMLTokenizer, HTMLSanitizerMixin):
def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True,
lowercaseElementName=True, lowercaseAttrName=True, **kwargs):
tokenizer.HTMLTokenizer.__init__(self, stream, encoding, parseMeta,
useChardet, lowercaseElementName,
lowercaseAttrName, **kwargs)
def __iter__(self):
for token in tokenizer.HTMLTokenizer.__iter__(self):
token = self.sanitize_token(token)
if token:
yield token
def sanitize_html(html):
"""Sanitizes an HTML fragment."""
p = html5lib.HTMLParser(tokenizer=HTMLSanitizer,
tree=treebuilders.getTreeBuilder("dom"))
dom_tree = p.parseFragment(html)
walker = treewalkers.getTreeWalker("dom")
stream = walker(dom_tree)
s = serializer.HTMLSerializer(omit_optional_tags=False,
quote_attr_values=True)
output_generator = s.serialize(stream)
return u''.join(output_generator)
def unescape(text):
"""source: http://effbot.org/zone/re-sub.htm#unescape-html
Removes HTML or XML character references and entities from a text string.
@param text The HTML (or XML) source text.
@return The plain text, as a Unicode string, if necessary.
"""
def fixup(m):
text = m.group(0)
if text[:2] == "&#":
# character reference
try:
if text[:3] == "&#x":
return unichr(int(text[3:-1], 16))
else:
return unichr(int(text[2:-1]))
except ValueError:
pass
else:
# named entity
try:
text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
except KeyError:
pass
return text # leave as is
return re.sub("&#?\w+;", fixup, text)
|
