Skip to content

Commit

Permalink
Use Unicode 15 tables for unicode normalization and segmentation
Browse files Browse the repository at this point in the history
To match with the Unicode support in Go 1.21, we'll now use the Unicode 15
tables when we're normalizing Unicode strings and when counting
user-perceived characters ("grapheme clusters") for source position
purposes.
  • Loading branch information
apparentlymart committed Aug 30, 2023
1 parent fef0406 commit edfd5c9
Show file tree
Hide file tree
Showing 8 changed files with 16 additions and 13 deletions.
7 changes: 4 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.18
require (
github.com/agext/levenshtein v1.2.1
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3
github.com/apparentlymart/go-textseg/v13 v13.0.0
github.com/apparentlymart/go-textseg/v15 v15.0.0
github.com/davecgh/go-spew v1.1.1
github.com/go-test/deep v1.0.3
github.com/google/go-cmp v0.3.1
Expand All @@ -20,10 +20,11 @@ require (
)

require (
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/kr/text v0.1.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.2.2 // indirect
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
golang.org/x/text v0.3.8 // indirect
golang.org/x/text v0.11.0 // indirect
)
10 changes: 6 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:o
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
Expand Down Expand Up @@ -40,12 +42,12 @@ golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
2 changes: 1 addition & 1 deletion hclsyntax/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"strconv"
"unicode/utf8"

"github.com/apparentlymart/go-textseg/v13/textseg"
"github.com/apparentlymart/go-textseg/v15/textseg"
"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"
)
Expand Down
2 changes: 1 addition & 1 deletion hclsyntax/parser_template.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"strings"
"unicode"

"github.com/apparentlymart/go-textseg/v13/textseg"
"github.com/apparentlymart/go-textseg/v15/textseg"
"github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"
)
Expand Down
2 changes: 1 addition & 1 deletion hclsyntax/token.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"bytes"
"fmt"

"github.com/apparentlymart/go-textseg/v13/textseg"
"github.com/apparentlymart/go-textseg/v15/textseg"
"github.com/hashicorp/hcl/v2"
)

Expand Down
2 changes: 1 addition & 1 deletion hclwrite/tokens.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"bytes"
"io"

"github.com/apparentlymart/go-textseg/v13/textseg"
"github.com/apparentlymart/go-textseg/v15/textseg"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
)
Expand Down
2 changes: 1 addition & 1 deletion json/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package json
import (
"fmt"

"github.com/apparentlymart/go-textseg/v13/textseg"
"github.com/apparentlymart/go-textseg/v15/textseg"
"github.com/hashicorp/hcl/v2"
)

Expand Down
2 changes: 1 addition & 1 deletion pos_scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"bufio"
"bytes"

"github.com/apparentlymart/go-textseg/v13/textseg"
"github.com/apparentlymart/go-textseg/v15/textseg"
)

// RangeScanner is a helper that will scan over a buffer using a bufio.SplitFunc
Expand Down

0 comments on commit edfd5c9

Please sign in to comment.