Fix html5lib#124: Move to webencodings for decoding the input byte st…

…ream.
gsnedders · Jan 12, 2016 · 85723e2 · 85723e2
1 parent 44b0bbc
commit 85723e2
Show file tree

Hide file tree

Showing 11 changed files with 49 additions and 306 deletions.
diff --git a/.pytest.expect b/.pytest.expect
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -22,6 +22,10 @@ Released on XXX
 
 * Move testsuite to ``py.test``.
 
+* Fix #124: move to webencodings for decoding the input byte stream;
+ this makes html5lib compliant with the Encoding Standard, and
+ introduces a required dependency on webencodings.
+
 
 0.9999999/1.0b8
 ~~~~~~~~~~~~~~~

diff --git a/html5lib/constants.py b/html5lib/constants.py
@@ -2846,235 +2846,6 @@
  0x9F: "\u0178",
 }
 
-encodings = {
- '437': 'cp437',
- '850': 'cp850',
- '852': 'cp852',
- '855': 'cp855',
- '857': 'cp857',
- '860': 'cp860',
- '861': 'cp861',
- '862': 'cp862',
- '863': 'cp863',
- '865': 'cp865',
- '866': 'cp866',
- '869': 'cp869',
- 'ansix341968': 'ascii',
- 'ansix341986': 'ascii',
- 'arabic': 'iso8859-6',
- 'ascii': 'ascii',
- 'asmo708': 'iso8859-6',
- 'big5': 'big5',
- 'big5hkscs': 'big5hkscs',
- 'chinese': 'gbk',
- 'cp037': 'cp037',
- 'cp1026': 'cp1026',
- 'cp154': 'ptcp154',
- 'cp367': 'ascii',
- 'cp424': 'cp424',
- 'cp437': 'cp437',
- 'cp500': 'cp500',
- 'cp775': 'cp775',
- 'cp819': 'windows-1252',
- 'cp850': 'cp850',
- 'cp852': 'cp852',
- 'cp855': 'cp855',
- 'cp857': 'cp857',
- 'cp860': 'cp860',
- 'cp861': 'cp861',
- 'cp862': 'cp862',
- 'cp863': 'cp863',
- 'cp864': 'cp864',
- 'cp865': 'cp865',
- 'cp866': 'cp866',
- 'cp869': 'cp869',
- 'cp936': 'gbk',
- 'cpgr': 'cp869',
- 'cpis': 'cp861',
- 'csascii': 'ascii',
- 'csbig5': 'big5',
- 'cseuckr': 'cp949',
- 'cseucpkdfmtjapanese': 'euc_jp',
- 'csgb2312': 'gbk',
- 'cshproman8': 'hp-roman8',
- 'csibm037': 'cp037',
- 'csibm1026': 'cp1026',
- 'csibm424': 'cp424',
- 'csibm500': 'cp500',
- 'csibm855': 'cp855',
- 'csibm857': 'cp857',
- 'csibm860': 'cp860',
- 'csibm861': 'cp861',
- 'csibm863': 'cp863',
- 'csibm864': 'cp864',
- 'csibm865': 'cp865',
- 'csibm866': 'cp866',
- 'csibm869': 'cp869',
- 'csiso2022jp': 'iso2022_jp',
- 'csiso2022jp2': 'iso2022_jp_2',
- 'csiso2022kr': 'iso2022_kr',
- 'csiso58gb231280': 'gbk',
- 'csisolatin1': 'windows-1252',
- 'csisolatin2': 'iso8859-2',
- 'csisolatin3': 'iso8859-3',
- 'csisolatin4': 'iso8859-4',
- 'csisolatin5': 'windows-1254',
- 'csisolatin6': 'iso8859-10',
- 'csisolatinarabic': 'iso8859-6',
- 'csisolatincyrillic': 'iso8859-5',
- 'csisolatingreek': 'iso8859-7',
- 'csisolatinhebrew': 'iso8859-8',
- 'cskoi8r': 'koi8-r',
- 'csksc56011987': 'cp949',
- 'cspc775baltic': 'cp775',
- 'cspc850multilingual': 'cp850',
- 'cspc862latinhebrew': 'cp862',
- 'cspc8codepage437': 'cp437',
- 'cspcp852': 'cp852',
- 'csptcp154': 'ptcp154',
- 'csshiftjis': 'shift_jis',
- 'csunicode11utf7': 'utf-7',
- 'cyrillic': 'iso8859-5',
- 'cyrillicasian': 'ptcp154',
- 'ebcdiccpbe': 'cp500',
- 'ebcdiccpca': 'cp037',
- 'ebcdiccpch': 'cp500',
- 'ebcdiccphe': 'cp424',
- 'ebcdiccpnl': 'cp037',
- 'ebcdiccpus': 'cp037',
- 'ebcdiccpwt': 'cp037',
- 'ecma114': 'iso8859-6',
- 'ecma118': 'iso8859-7',
- 'elot928': 'iso8859-7',
- 'eucjp': 'euc_jp',
- 'euckr': 'cp949',
- 'extendedunixcodepackedformatforjapanese': 'euc_jp',
- 'gb18030': 'gb18030',
- 'gb2312': 'gbk',
- 'gb231280': 'gbk',
- 'gbk': 'gbk',
- 'greek': 'iso8859-7',
- 'greek8': 'iso8859-7',
- 'hebrew': 'iso8859-8',
- 'hproman8': 'hp-roman8',
- 'hzgb2312': 'hz',
- 'ibm037': 'cp037',
- 'ibm1026': 'cp1026',
- 'ibm367': 'ascii',
- 'ibm424': 'cp424',
- 'ibm437': 'cp437',
- 'ibm500': 'cp500',
- 'ibm775': 'cp775',
- 'ibm819': 'windows-1252',
- 'ibm850': 'cp850',
- 'ibm852': 'cp852',
- 'ibm855': 'cp855',
- 'ibm857': 'cp857',
- 'ibm860': 'cp860',
- 'ibm861': 'cp861',
- 'ibm862': 'cp862',
- 'ibm863': 'cp863',
- 'ibm864': 'cp864',
- 'ibm865': 'cp865',
- 'ibm866': 'cp866',
- 'ibm869': 'cp869',
- 'iso2022jp': 'iso2022_jp',
- 'iso2022jp2': 'iso2022_jp_2',
- 'iso2022kr': 'iso2022_kr',
- 'iso646irv1991': 'ascii',
- 'iso646us': 'ascii',
- 'iso88591': 'windows-1252',
- 'iso885910': 'iso8859-10',
- 'iso8859101992': 'iso8859-10',
- 'iso885911987': 'windows-1252',
- 'iso885913': 'iso8859-13',
- 'iso885914': 'iso8859-14',
- 'iso8859141998': 'iso8859-14',
- 'iso885915': 'iso8859-15',
- 'iso885916': 'iso8859-16',
- 'iso8859162001': 'iso8859-16',
- 'iso88592': 'iso8859-2',
- 'iso885921987': 'iso8859-2',
- 'iso88593': 'iso8859-3',
- 'iso885931988': 'iso8859-3',
- 'iso88594': 'iso8859-4',
- 'iso885941988': 'iso8859-4',
- 'iso88595': 'iso8859-5',
- 'iso885951988': 'iso8859-5',
- 'iso88596': 'iso8859-6',
- 'iso885961987': 'iso8859-6',
- 'iso88597': 'iso8859-7',
- 'iso885971987': 'iso8859-7',
- 'iso88598': 'iso8859-8',
- 'iso885981988': 'iso8859-8',
- 'iso88599': 'windows-1254',
- 'iso885991989': 'windows-1254',
- 'isoceltic': 'iso8859-14',
- 'isoir100': 'windows-1252',
- 'isoir101': 'iso8859-2',
- 'isoir109': 'iso8859-3',
- 'isoir110': 'iso8859-4',
- 'isoir126': 'iso8859-7',
- 'isoir127': 'iso8859-6',
- 'isoir138': 'iso8859-8',
- 'isoir144': 'iso8859-5',
- 'isoir148': 'windows-1254',
- 'isoir149': 'cp949',
- 'isoir157': 'iso8859-10',
- 'isoir199': 'iso8859-14',
- 'isoir226': 'iso8859-16',
- 'isoir58': 'gbk',
- 'isoir6': 'ascii',
- 'koi8r': 'koi8-r',
- 'koi8u': 'koi8-u',
- 'korean': 'cp949',
- 'ksc5601': 'cp949',
- 'ksc56011987': 'cp949',
- 'ksc56011989': 'cp949',
- 'l1': 'windows-1252',
- 'l10': 'iso8859-16',
- 'l2': 'iso8859-2',
- 'l3': 'iso8859-3',
- 'l4': 'iso8859-4',
- 'l5': 'windows-1254',
- 'l6': 'iso8859-10',
- 'l8': 'iso8859-14',
- 'latin1': 'windows-1252',
- 'latin10': 'iso8859-16',
- 'latin2': 'iso8859-2',
- 'latin3': 'iso8859-3',
- 'latin4': 'iso8859-4',
- 'latin5': 'windows-1254',
- 'latin6': 'iso8859-10',
- 'latin8': 'iso8859-14',
- 'latin9': 'iso8859-15',
- 'ms936': 'gbk',
- 'mskanji': 'shift_jis',
- 'pt154': 'ptcp154',
- 'ptcp154': 'ptcp154',
- 'r8': 'hp-roman8',
- 'roman8': 'hp-roman8',
- 'shiftjis': 'shift_jis',
- 'tis620': 'cp874',
- 'unicode11utf7': 'utf-7',
- 'us': 'ascii',
- 'usascii': 'ascii',
- 'utf16': 'utf-16',
- 'utf16be': 'utf-16-be',
- 'utf16le': 'utf-16-le',
- 'utf8': 'utf-8',
- 'windows1250': 'cp1250',
- 'windows1251': 'cp1251',
- 'windows1252': 'cp1252',
- 'windows1253': 'cp1253',
- 'windows1254': 'cp1254',
- 'windows1255': 'cp1255',
- 'windows1256': 'cp1256',
- 'windows1257': 'cp1257',
- 'windows1258': 'cp1258',
- 'windows936': 'gbk',
- 'x-x-big5': 'big5'}
-
 tokenTypes = {
  "Doctype": 0,
  "Characters": 1,

diff --git a/html5lib/html5parser.py b/html5lib/html5parser.py
@@ -139,7 +139,7 @@ def documentEncoding(self):
  """
  if not hasattr(self, 'tokenizer'):
  return None
- return self.tokenizer.stream.charEncoding[0]
+ return self.tokenizer.stream.charEncoding[0].name
 
  def isHTMLIntegrationPoint(self, element):
  if (element.name == "annotation-xml" and