forked from xonsh/xonsh
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test_lexer.py
169 lines (134 loc) · 4.84 KB
/
test_lexer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
"""Tests the xonsh lexer."""
from __future__ import unicode_literals, print_function
import os
import sys
from collections import Sequence
from pprint import pprint, pformat
sys.path.insert(0, os.path.abspath('..')) # FIXME
import nose
from nose.tools import assert_equal
from ply.lex import LexToken
from xonsh.lexer import Lexer
LEXER_ARGS = {'lextab': 'lexer_test_table', 'debug': 0}
def ensure_tuple(x):
if isinstance(x, LexToken):
# line numbers can no longer be solely determined from the lexer
#x = (x.type, x.value, x.lineno, x.lexpos)
x = (x.type, x.value, x.lexpos)
elif isinstance(x, tuple):
pass
elif isinstance(x, Sequence):
x = tuple(x)
else:
raise TypeError('{0} is not a sequence'.format(x))
return x
def tokens_equal(x, y):
"""Tests whether two token are equal."""
xtup = ensure_tuple(x)
ytup = ensure_tuple(y)
return xtup == ytup
def assert_token_equal(x, y):
"""Asserts that two tokens are equal."""
if not tokens_equal(x, y):
msg = 'The tokens differ: {0!r} != {1!r}'.format(x, y)
raise AssertionError(msg)
def assert_tokens_equal(x, y):
"""Asserts that two token sequences are equal."""
if len(x) != len(y):
msg = 'The tokens sequences have different lengths: {0!r} != {1!r}\n'
msg += '# x\n{2}\n\n# y\n{3}'
raise AssertionError(msg.format(len(x), len(y), pformat(x), pformat(y)))
diffs = []
diffs = [(a, b) for a, b in zip(x, y) if not tokens_equal(a, b)]
if len(diffs) > 0:
msg = ['The token sequnces differ: ']
for a, b in diffs:
msg += ['', '- ' + repr(a), '+ ' + repr(b)]
msg = '\n'.join(msg)
raise AssertionError(msg)
def check_token(input, exp):
l = Lexer()
l.build(**LEXER_ARGS)
l.input(input)
obs = list(l)
if len(obs) != 1:
msg = 'The observed sequence does not have length-1: {0!r} != 1\n'
msg += '# obs\n{1}'
raise AssertionError(msg.format(len(obs), pformat(obs)))
assert_token_equal(exp, obs[0])
def check_tokens(input, exp):
l = Lexer()
l.build(**LEXER_ARGS)
l.input(input)
obs = list(l)
assert_tokens_equal(exp, obs)
def test_int_literal():
yield check_token, '42', ['INT_LITERAL', 42, 0]
def test_hex_literal():
yield check_token, '0x42', ['HEX_LITERAL', int('0x42', 16), 0]
def test_oct_o_literal():
yield check_token, '0o42', ['OCT_LITERAL', int('0o42', 8), 0]
def test_oct_no_o_literal():
yield check_token, '042', ['OCT_LITERAL', int('042', 8), 0]
def test_bin_literal():
yield check_token, '0b101010', ['BIN_LITERAL', int('0b101010', 2), 0]
def test_indent():
exp = [('INDENT', ' \t ', 0),
('INT_LITERAL', 42, 5)]
yield check_tokens, ' \t 42', exp
def test_post_whitespace():
input = '42 \t '
exp = [('INT_LITERAL', 42, 0)]
yield check_tokens, input, exp
def test_internal_whitespace():
input = '42 +\t65'
exp = [('INT_LITERAL', 42, 0),
('PLUS', '+', 4),
('INT_LITERAL', 65, 6),]
yield check_tokens, input, exp
def test_indent_internal_whitespace():
input = ' 42 +\t65'
exp = [('INDENT', ' ', 0),
('INT_LITERAL', 42, 1),
('PLUS', '+', 5),
('INT_LITERAL', 65, 7),]
yield check_tokens, input, exp
def test_assignment():
input = 'x = 42'
exp = [('NAME', 'x', 0),
('EQUALS', '=', 2),
('INT_LITERAL', 42, 4),]
yield check_tokens, input, exp
def test_multiline():
input = 'x\ny'
exp = [('NAME', 'x', 0),
('NEWLINE', '\n', 1),
('NAME', 'y', 2),]
yield check_tokens, input, exp
def test_and():
yield check_token, 'and', ['AND', 'and', 0]
def test_single_quote_literal():
yield check_token, "'yo'", ['STRING_LITERAL', "'yo'", 0]
def test_double_quote_literal():
yield check_token, '"yo"', ['STRING_LITERAL', '"yo"', 0]
def test_triple_single_quote_literal():
yield check_token, "'''yo'''", ['STRING_LITERAL', "'''yo'''", 0]
def test_triple_double_quote_literal():
yield check_token, '"""yo"""', ['STRING_LITERAL', '"""yo"""', 0]
def test_single_raw_string_literal():
yield check_token, "r'yo'", ['RAW_STRING_LITERAL', "r'yo'", 0]
def test_double_raw_string_literal():
yield check_token, 'r"yo"', ['RAW_STRING_LITERAL', 'r"yo"', 0]
def test_single_unicode_literal():
yield check_token, "u'yo'", ['UNICODE_LITERAL', "u'yo'", 0]
def test_double_unicode_literal():
yield check_token, 'u"yo"', ['UNICODE_LITERAL', 'u"yo"', 0]
def test_single_bytes_literal():
yield check_token, "b'yo'", ['BYTES_LITERAL', "b'yo'", 0]
def test_float_literals():
cases = ['0.0', '.0', '0.', '1e10', '1.e42', '0.1e42', '0.5e-42',
'5E10', '5e+42']
for s in cases:
yield check_token, s, ['FLOAT_LITERAL', float(s), 0]
if __name__ == '__main__':
nose.runmodule()