forked from hashicorp/hcl
-
Notifications
You must be signed in to change notification settings - Fork 0
/
tokens.go
170 lines (147 loc) · 4.55 KB
/
tokens.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
package hclwrite
import (
"bytes"
"io"
"github.com/apparentlymart/go-textseg/textseg"
"github.com/hashicorp/hcl2/hcl/hclsyntax"
)
// TokenGen is an abstract type that can append tokens to a list. It is the
// low-level foundation underlying the hclwrite AST; the AST provides a
// convenient abstraction over raw token sequences to facilitate common tasks,
// but it's also possible to directly manipulate the tree of token generators
// to make changes that the AST API doesn't directly allow.
type TokenGen interface {
EachToken(TokenCallback)
}
// TokenCallback is used with TokenGen implementations to specify the action
// that is to be taken for each token in the flattened token sequence.
type TokenCallback func(*Token)
// Token is a single sequence of bytes annotated with a type. It is similar
// in purpose to hclsyntax.Token, but discards the source position information
// since that is not useful in code generation.
type Token struct {
Type hclsyntax.TokenType
Bytes []byte
// We record the number of spaces before each token so that we can
// reproduce the exact layout of the original file when we're making
// surgical changes in-place. When _new_ code is created it will always
// be in the canonical style, but we preserve layout of existing code.
SpacesBefore int
}
// Tokens is a flat list of tokens.
type Tokens []*Token
func (ts Tokens) WriteTo(wr io.Writer) (int, error) {
seq := &TokenSeq{ts}
return seq.WriteTo(wr)
}
func (ts Tokens) Bytes() []byte {
buf := &bytes.Buffer{}
ts.WriteTo(buf)
return buf.Bytes()
}
// Columns returns the number of columns (grapheme clusters) the token sequence
// occupies. The result is not meaningful if there are newline or single-line
// comment tokens in the sequence.
func (ts Tokens) Columns() int {
ret := 0
for _, token := range ts {
ret += token.SpacesBefore // spaces are always worth one column each
ct, _ := textseg.TokenCount(token.Bytes, textseg.ScanGraphemeClusters)
ret += ct
}
return ret
}
// TokenSeq combines zero or more TokenGens together to produce a flat sequence
// of tokens from a tree of TokenGens.
type TokenSeq []TokenGen
func (t *Token) EachToken(cb TokenCallback) {
cb(t)
}
func (ts Tokens) EachToken(cb TokenCallback) {
for _, t := range ts {
cb(t)
}
}
func (ts *TokenSeq) EachToken(cb TokenCallback) {
if ts == nil {
return
}
for _, gen := range *ts {
gen.EachToken(cb)
}
}
// Tokens returns the flat list of tokens represented by the receiving
// token sequence.
func (ts *TokenSeq) Tokens() Tokens {
var tokens Tokens
ts.EachToken(func(token *Token) {
tokens = append(tokens, token)
})
return tokens
}
// WriteTo takes an io.Writer and writes the bytes for each token to it,
// along with the spacing that separates each token. In other words, this
// allows serializing the tokens to a file or other such byte stream.
func (ts *TokenSeq) WriteTo(wr io.Writer) (int, error) {
// We know we're going to be writing a lot of small chunks of repeated
// space characters, so we'll prepare a buffer of these that we can
// easily pass to wr.Write without any further allocation.
spaces := make([]byte, 40)
for i := range spaces {
spaces[i] = ' '
}
var n int
var err error
ts.EachToken(func(token *Token) {
if err != nil {
return
}
for spacesBefore := token.SpacesBefore; spacesBefore > 0; spacesBefore -= len(spaces) {
thisChunk := spacesBefore
if thisChunk > len(spaces) {
thisChunk = len(spaces)
}
var thisN int
thisN, err = wr.Write(spaces[:thisChunk])
n += thisN
if err != nil {
return
}
}
var thisN int
thisN, err = wr.Write(token.Bytes)
n += thisN
})
return n, err
}
// SoloToken returns the single token represented by the receiving sequence,
// or nil if the sequence does not represent exactly one token.
func (ts *TokenSeq) SoloToken() *Token {
var ret *Token
found := false
ts.EachToken(func(tok *Token) {
if ret == nil && !found {
ret = tok
found = true
} else if ret != nil && found {
ret = nil
}
})
return ret
}
// IsIdent returns true if and only if the token sequence represents a single
// ident token whose name matches the given string.
func (ts *TokenSeq) IsIdent(name []byte) bool {
tok := ts.SoloToken()
if tok == nil {
return false
}
if tok.Type != hclsyntax.TokenIdent {
return false
}
return bytes.Equal(tok.Bytes, name)
}
// TokenSeqEmpty is a TokenSeq that contains no tokens. It can be used anywhere,
// but its primary purpose is to be assigned as a replacement for a non-empty
// TokenSeq when eliminating a section of an input file.
var TokenSeqEmpty = TokenSeq([]TokenGen(nil))