-
Notifications
You must be signed in to change notification settings - Fork 3
/
token.go
180 lines (157 loc) · 3.37 KB
/
token.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
package nstcc
import (
"bytes"
"errors"
)
func dup(s []byte) []byte {
t := make([]byte, len(s))
copy(t, s)
return t
}
const (
hashInit = 1
hashSize = 8192
)
func hashFunc(h uint32, c uint8) uint32 {
return h*263 + uint32(c)
}
type idents struct {
list []*tokenSym
hash [hashSize]*tokenSym
}
func (m *idents) init() {
i := uint32(0)
for _, j := range builtInTokensLengths {
m.byStr(builtInTokensStrings[i:j])
i = j
}
}
func (m *idents) byTok(t token) *tokenSym {
return m.list[t-tokIdent]
}
// The TCC code calls this tok_alloc.
func (m *idents) byStr(s []byte) (*tokenSym, error) {
h := uint32(hashInit)
for _, c := range s {
h = hashFunc(h, c)
}
p := &m.hash[h&(hashSize-1)]
for {
y := *p
if y == nil {
break
}
if bytes.Equal(s, y.str) {
return y, nil
}
p = &y.hashNext
}
return m.alloc(p, s)
}
// The TCC code calls this tok_alloc_new.
func (m *idents) alloc(p **tokenSym, s []byte) (*tokenSym, error) {
t := tokIdent + token(len(m.list))
if t >= symFirstAnon {
return nil, errors.New("nstcc: memory full")
}
y := &tokenSym{
tok: t,
str: dup(s),
}
m.list = append(m.list, y)
*p = y
return y, nil
}
func (m *idents) defineFind(t token) *sym {
t -= tokIdent
if t < 0 || len(m.list) <= int(t) {
return nil
}
return m.list[t].symDefine
}
func (m *idents) defineUndef(s *sym) {
t := s.tok - tokIdent
if 0 <= t && int(t) < len(m.list) {
m.list[t].symDefine = nil
}
s.tok = 0
}
type token int32
func (t token) isSpace() bool {
return t == ' ' || t == '\t' || t == '\v' || t == '\f' || t == '\r'
}
type tokenValue struct {
tok token
tokc cValue
}
type tokenString struct {
tokStr []tokenValue
lastLineNum int
}
type macroLevel struct {
prev *macroLevel
tokStr []tokenValue
}
const (
tokEOF token = -1 // End Of File.
tokEOM token = -2 // End Of Macro.
tokULT token = 0x92
tokUGE token = 0x93
tokEq token = 0x94
tokNE token = 0x95
tokULE token = 0x96
tokUGT token = 0x97
tokNset token = 0x98
tokNclear token = 0x99
tokLT token = 0x9c
tokGE token = 0x9d
tokLE token = 0x9e
tokGT token = 0x9f
tokLAnd token = 0xa0
tokLOr token = 0xa1
tokDec token = 0xa2
tokMid token = 0xa3
tokInc token = 0xa4
tokUDiv token = 0xb0
tokUMod token = 0xb1
tokPDiv token = 0xb2
tokCInt token = 0xb3
tokCChar token = 0xb4
tokStr token = 0xb5
tokTwoSharps token = 0xb6
tokLChar token = 0xb7
tokLStr token = 0xb8
tokCFloat token = 0xb9
tokLineNum token = 0xba
tokCDouble token = 0xc0
tokCLDouble token = 0xc1
tokUMull token = 0xc2
tokAddC1 token = 0xc3
tokAddC2 token = 0xc4
tokSubC1 token = 0xc5
tokSubC2 token = 0xc6
tokCUint token = 0xc8
tokCLLong token = 0xc9
tokCULLong token = 0xca
tokArrow token = 0xcb
tokDots token = 0xcc
tokShR token = 0xcd
tokPPNum token = 0xce
tokNoSubst token = 0xcf
tokShL token = 0x01
tokSAR token = 0x02
tokAMod token = 0xa5
tokAAnd token = 0xa6
tokAMul token = 0xaa
tokAAdd token = 0xab
tokASub token = 0xad
tokADiv token = 0xaf
tokAXor token = 0xde
tokAOr token = 0xfc
tokAShL token = 0x81
tokASAR token = 0x82
// tokIdent is the token value of the first identifier token. Token values
// greater than or equal to tokIdent represent identifiers. Token values
// less than tokIdent represent symbols.
tokIdent token = 0x100
)