-
Notifications
You must be signed in to change notification settings - Fork 0
/
js_tokenizer.py
35 lines (30 loc) · 856 Bytes
/
js_tokenizer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
from slimit.lexer import Lexer
CodeExample = \
"""// 这是我随便写的注释
var browser=navigator.appName
var b_version=navigator.appVersion
var version=parseFloat(b_version)
document.write("浏览器名称:"+ browser)
document.write("<br />")
document.write("浏览器版本:"+ version)"""
def tokenize(code, need_type_info=False):
lexer = Lexer()
lexer.input(code)
tokens = []
types = []
for token in lexer:
#print(token)
tokens.append(token.value)
types.append(token.type)
if need_type_info:
return tokens, types
else:
return tokens
if __name__=="__main__":
print("No code/function passed in, function below is used to show you a case:")
print()
print(CodeExample)
print()
tokens, types = tokenize(CodeExample, True)
print(tokens)
print(types)