From 4235ca04c06fb1028aba2841aa587c1b3d598976 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Thu, 24 Mar 2022 09:35:14 +0900 Subject: [PATCH 01/43] fix: an incompatible behavior on map key decoder fix #342 The map key decoder has an incompatible behavior when the type kind is string and the type has UnmarshalJSON. --- decode_test.go | 18 ++++++++++++++++++ internal/decoder/compile.go | 3 +++ 2 files changed, 21 insertions(+) diff --git a/decode_test.go b/decode_test.go index 3820173a..57d9c6a4 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3889,3 +3889,21 @@ func TestIssue348(t *testing.T) { } } } + +type issue342 string + +func (t *issue342) UnmarshalJSON(b []byte) error { + panic("unreachable") +} + +func TestIssue342(t *testing.T) { + var v map[issue342]int + in := []byte(`{"a":1}`) + if err := json.Unmarshal(in, &v); err != nil { + t.Errorf("unexpected error: %v", err) + } + expected := 1 + if got := v["a"]; got != expected { + t.Errorf("unexpected result: got(%v) != expected(%v)", got, expected) + } +} diff --git a/internal/decoder/compile.go b/internal/decoder/compile.go index b2e9f57c..bf17d1b2 100644 --- a/internal/decoder/compile.go +++ b/internal/decoder/compile.go @@ -154,6 +154,9 @@ func compileMapKey(typ *runtime.Type, structName, fieldName string, structTypeTo if runtime.PtrTo(typ).Implements(unmarshalTextType) { return newUnmarshalTextDecoder(runtime.PtrTo(typ), structName, fieldName), nil } + if typ.Kind() == reflect.String { + return newStringDecoder(structName, fieldName), nil + } dec, err := compile(typ, structName, fieldName, structTypeToDecoder) if err != nil { return nil, err From e43fb0f99061586d663821db2aafb56140f2cdec Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Fri, 25 Mar 2022 03:03:25 +0900 Subject: [PATCH 02/43] fix: add filtering on slow path --- internal/encoder/compiler_norace.go | 6 +++++- internal/encoder/compiler_race.go | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/encoder/compiler_norace.go b/internal/encoder/compiler_norace.go index 706c3dbc..20c93cbf 100644 --- a/internal/encoder/compiler_norace.go +++ b/internal/encoder/compiler_norace.go @@ -5,7 +5,11 @@ package encoder func CompileToGetCodeSet(ctx *RuntimeContext, typeptr uintptr) (*OpcodeSet, error) { if typeptr > typeAddr.MaxTypeAddr || typeptr < typeAddr.BaseTypeAddr { - return compileToGetCodeSetSlowPath(typeptr) + codeSet, err := compileToGetCodeSetSlowPath(typeptr) + if err != nil { + return nil, err + } + return getFilteredCodeSetIfNeeded(ctx, codeSet) } index := (typeptr - typeAddr.BaseTypeAddr) >> typeAddr.AddrShift if codeSet := cachedOpcodeSets[index]; codeSet != nil { diff --git a/internal/encoder/compiler_race.go b/internal/encoder/compiler_race.go index 2589ef1a..13ba23fd 100644 --- a/internal/encoder/compiler_race.go +++ b/internal/encoder/compiler_race.go @@ -11,7 +11,11 @@ var setsMu sync.RWMutex func CompileToGetCodeSet(ctx *RuntimeContext, typeptr uintptr) (*OpcodeSet, error) { if typeptr > typeAddr.MaxTypeAddr || typeptr < typeAddr.BaseTypeAddr { - return compileToGetCodeSetSlowPath(typeptr) + codeSet, err := compileToGetCodeSetSlowPath(typeptr) + if err != nil { + return nil, err + } + return getFilteredCodeSetIfNeeded(ctx, codeSet) } index := (typeptr - typeAddr.BaseTypeAddr) >> typeAddr.AddrShift setsMu.RLock() From 321fe31260aebde95ec75defa625967893cd8860 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Fri, 25 Mar 2022 05:09:22 +0900 Subject: [PATCH 03/43] feat: add DebugWith option --- encode.go | 2 ++ encode_test.go | 3 ++- internal/encoder/option.go | 6 +++++- internal/encoder/vm/debug_vm.go | 21 ++++++++++---------- internal/encoder/vm_color/debug_vm.go | 21 ++++++++++---------- internal/encoder/vm_color_indent/debug_vm.go | 21 ++++++++++---------- internal/encoder/vm_indent/debug_vm.go | 21 ++++++++++---------- option.go | 9 +++++++++ 8 files changed, 62 insertions(+), 42 deletions(-) diff --git a/encode.go b/encode.go index c9527c0e..4bd899f3 100644 --- a/encode.go +++ b/encode.go @@ -3,6 +3,7 @@ package json import ( "context" "io" + "os" "unsafe" "github.com/goccy/go-json/internal/encoder" @@ -62,6 +63,7 @@ func (e *Encoder) encodeWithOption(ctx *encoder.RuntimeContext, v interface{}, o ctx.Option.Flag |= encoder.HTMLEscapeOption } ctx.Option.Flag |= encoder.NormalizeUTF8Option + ctx.Option.DebugOut = os.Stdout for _, optFunc := range optFuncs { optFunc(ctx.Option) } diff --git a/encode_test.go b/encode_test.go index 469b6551..92f43043 100644 --- a/encode_test.go +++ b/encode_test.go @@ -477,7 +477,8 @@ func TestDebugMode(t *testing.T) { t.Fatal("expected error") } }() - json.MarshalWithOption(mustErrTypeForDebug{}, json.Debug()) + var buf bytes.Buffer + json.MarshalWithOption(mustErrTypeForDebug{}, json.Debug(), json.DebugWith(&buf)) } func TestIssue116(t *testing.T) { diff --git a/internal/encoder/option.go b/internal/encoder/option.go index dcec8f20..82d5ce3e 100644 --- a/internal/encoder/option.go +++ b/internal/encoder/option.go @@ -1,6 +1,9 @@ package encoder -import "context" +import ( + "context" + "io" +) type OptionFlag uint8 @@ -19,6 +22,7 @@ type Option struct { Flag OptionFlag ColorScheme *ColorScheme Context context.Context + DebugOut io.Writer } type EncodeFormat struct { diff --git a/internal/encoder/vm/debug_vm.go b/internal/encoder/vm/debug_vm.go index 05509fed..fbbc0de4 100644 --- a/internal/encoder/vm/debug_vm.go +++ b/internal/encoder/vm/debug_vm.go @@ -16,16 +16,17 @@ func DebugRun(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) } if err := recover(); err != nil { - fmt.Println("=============[DEBUG]===============") - fmt.Println("* [TYPE]") - fmt.Println(codeSet.Type) - fmt.Printf("\n") - fmt.Println("* [ALL OPCODE]") - fmt.Println(code.Dump()) - fmt.Printf("\n") - fmt.Println("* [CONTEXT]") - fmt.Printf("%+v\n", ctx) - fmt.Println("===================================") + w := ctx.Option.DebugOut + fmt.Fprintln(w, "=============[DEBUG]===============") + fmt.Fprintln(w, "* [TYPE]") + fmt.Fprintln(w, codeSet.Type) + fmt.Fprintf(w, "\n") + fmt.Fprintln(w, "* [ALL OPCODE]") + fmt.Fprintln(w, code.Dump()) + fmt.Fprintf(w, "\n") + fmt.Fprintln(w, "* [CONTEXT]") + fmt.Fprintf(w, "%+v\n", ctx) + fmt.Fprintln(w, "===================================") panic(err) } }() diff --git a/internal/encoder/vm_color/debug_vm.go b/internal/encoder/vm_color/debug_vm.go index 6a6a33d2..925f61ed 100644 --- a/internal/encoder/vm_color/debug_vm.go +++ b/internal/encoder/vm_color/debug_vm.go @@ -16,16 +16,17 @@ func DebugRun(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) defer func() { if err := recover(); err != nil { - fmt.Println("=============[DEBUG]===============") - fmt.Println("* [TYPE]") - fmt.Println(codeSet.Type) - fmt.Printf("\n") - fmt.Println("* [ALL OPCODE]") - fmt.Println(code.Dump()) - fmt.Printf("\n") - fmt.Println("* [CONTEXT]") - fmt.Printf("%+v\n", ctx) - fmt.Println("===================================") + w := ctx.Option.DebugOut + fmt.Fprintln(w, "=============[DEBUG]===============") + fmt.Fprintln(w, "* [TYPE]") + fmt.Fprintln(w, codeSet.Type) + fmt.Fprintf(w, "\n") + fmt.Fprintln(w, "* [ALL OPCODE]") + fmt.Fprintln(w, code.Dump()) + fmt.Fprintf(w, "\n") + fmt.Fprintln(w, "* [CONTEXT]") + fmt.Fprintf(w, "%+v\n", ctx) + fmt.Fprintln(w, "===================================") panic(err) } }() diff --git a/internal/encoder/vm_color_indent/debug_vm.go b/internal/encoder/vm_color_indent/debug_vm.go index a68bbf6b..dd4cd489 100644 --- a/internal/encoder/vm_color_indent/debug_vm.go +++ b/internal/encoder/vm_color_indent/debug_vm.go @@ -16,16 +16,17 @@ func DebugRun(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) defer func() { if err := recover(); err != nil { - fmt.Println("=============[DEBUG]===============") - fmt.Println("* [TYPE]") - fmt.Println(codeSet.Type) - fmt.Printf("\n") - fmt.Println("* [ALL OPCODE]") - fmt.Println(code.Dump()) - fmt.Printf("\n") - fmt.Println("* [CONTEXT]") - fmt.Printf("%+v\n", ctx) - fmt.Println("===================================") + w := ctx.Option.DebugOut + fmt.Fprintln(w, "=============[DEBUG]===============") + fmt.Fprintln(w, "* [TYPE]") + fmt.Fprintln(w, codeSet.Type) + fmt.Fprintf(w, "\n") + fmt.Fprintln(w, "* [ALL OPCODE]") + fmt.Fprintln(w, code.Dump()) + fmt.Fprintf(w, "\n") + fmt.Fprintln(w, "* [CONTEXT]") + fmt.Fprintf(w, "%+v\n", ctx) + fmt.Fprintln(w, "===================================") panic(err) } }() diff --git a/internal/encoder/vm_indent/debug_vm.go b/internal/encoder/vm_indent/debug_vm.go index 4cfd17ab..99395388 100644 --- a/internal/encoder/vm_indent/debug_vm.go +++ b/internal/encoder/vm_indent/debug_vm.go @@ -16,16 +16,17 @@ func DebugRun(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) defer func() { if err := recover(); err != nil { - fmt.Println("=============[DEBUG]===============") - fmt.Println("* [TYPE]") - fmt.Println(codeSet.Type) - fmt.Printf("\n") - fmt.Println("* [ALL OPCODE]") - fmt.Println(code.Dump()) - fmt.Printf("\n") - fmt.Println("* [CONTEXT]") - fmt.Printf("%+v\n", ctx) - fmt.Println("===================================") + w := ctx.Option.DebugOut + fmt.Fprintln(w, "=============[DEBUG]===============") + fmt.Fprintln(w, "* [TYPE]") + fmt.Fprintln(w, codeSet.Type) + fmt.Fprintf(w, "\n") + fmt.Fprintln(w, "* [ALL OPCODE]") + fmt.Fprintln(w, code.Dump()) + fmt.Fprintf(w, "\n") + fmt.Fprintln(w, "* [CONTEXT]") + fmt.Fprintf(w, "%+v\n", ctx) + fmt.Fprintln(w, "===================================") panic(err) } }() diff --git a/option.go b/option.go index 0138d777..af400a45 100644 --- a/option.go +++ b/option.go @@ -1,6 +1,8 @@ package json import ( + "io" + "github.com/goccy/go-json/internal/decoder" "github.com/goccy/go-json/internal/encoder" ) @@ -39,6 +41,13 @@ func Debug() EncodeOptionFunc { } } +// DebugWith sets the destination to write debug messages. +func DebugWith(w io.Writer) EncodeOptionFunc { + return func(opt *EncodeOption) { + opt.DebugOut = w + } +} + // Colorize add an identifier for coloring to the string of the encoded result. func Colorize(scheme *ColorScheme) EncodeOptionFunc { return func(opt *EncodeOption) { From d9df77a119f49379052e524092c824f3a07701c7 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Thu, 7 Apr 2022 18:10:49 +0900 Subject: [PATCH 04/43] fix: add a fallback uint8 sliceDecoder to bytesDecoder fix #360 --- decode_test.go | 11 +++++++++++ internal/decoder/bytes.go | 5 ++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/decode_test.go b/decode_test.go index 57d9c6a4..577d1d52 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3907,3 +3907,14 @@ func TestIssue342(t *testing.T) { t.Errorf("unexpected result: got(%v) != expected(%v)", got, expected) } } + +func TestIssue360(t *testing.T) { + var uints []uint8 + err := json.Unmarshal([]byte(`[0, 1, 2]`), &uints) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if len(uints) != 3 || !(uints[0] == 0 && uints[1] == 1 && uints[2] == 2) { + t.Errorf("unexpected result: %v", uints) + } +} diff --git a/internal/decoder/bytes.go b/internal/decoder/bytes.go index 01a37fef..92c7dcf6 100644 --- a/internal/decoder/bytes.go +++ b/internal/decoder/bytes.go @@ -23,9 +23,8 @@ func byteUnmarshalerSliceDecoder(typ *runtime.Type, structName string, fieldName unmarshalDecoder = newUnmarshalJSONDecoder(runtime.PtrTo(typ), structName, fieldName) case runtime.PtrTo(typ).Implements(unmarshalTextType): unmarshalDecoder = newUnmarshalTextDecoder(runtime.PtrTo(typ), structName, fieldName) - } - if unmarshalDecoder == nil { - return nil + default: + unmarshalDecoder, _ = compileUint8(typ, structName, fieldName) } return newSliceDecoder(unmarshalDecoder, typ, 1, structName, fieldName) } From 5c2b1916eb29ad33792f412c6d6cf5b3a9a796b1 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Mon, 11 Apr 2022 18:28:11 +0900 Subject: [PATCH 05/43] chore: update golangci-lint version --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index eb441f7f..684cbc76 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,5 +12,5 @@ jobs: - uses: actions/checkout@v2 - uses: golangci/golangci-lint-action@v2 with: - version: v1.36.0 + version: v1.45.2 args: --timeout=5m From 4311bab3dc77a4f428d80b18d23aa5a04867ac4d Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Mon, 11 Apr 2022 20:43:58 +0900 Subject: [PATCH 06/43] style: go fmt --- internal/decoder/compile_norace.go | 1 + internal/decoder/compile_race.go | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/decoder/compile_norace.go b/internal/decoder/compile_norace.go index 592f6373..eb7e2b13 100644 --- a/internal/decoder/compile_norace.go +++ b/internal/decoder/compile_norace.go @@ -1,3 +1,4 @@ +//go:build !race // +build !race package decoder diff --git a/internal/decoder/compile_race.go b/internal/decoder/compile_race.go index b691bc94..49cdda4a 100644 --- a/internal/decoder/compile_race.go +++ b/internal/decoder/compile_race.go @@ -1,3 +1,4 @@ +//go:build race // +build race package decoder From 0da28e819a8b9597213fc7aa23a55990c4c96ac2 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Tue, 12 Apr 2022 12:47:36 +0900 Subject: [PATCH 07/43] chore: add disable --- .golangci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 44ae40f6..57ae5a52 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -48,6 +48,14 @@ linters: - nlreturn - testpackage - wsl + - varnamelen + - nilnil + - ireturn + - govet + - forcetypeassert + - cyclop + - containedctx + - revive issues: exclude-rules: From 6db1acfcb64d486e947112239fafeef113a95890 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Fri, 22 Apr 2022 00:35:58 +0900 Subject: [PATCH 08/43] fix: to care about the case of OpInterfacePtr fix #359 --- decode_test.go | 13 +++++++++++++ internal/encoder/opcode.go | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/decode_test.go b/decode_test.go index 577d1d52..1d9f47c5 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3918,3 +3918,16 @@ func TestIssue360(t *testing.T) { t.Errorf("unexpected result: %v", uints) } } + +func TestIssue359(t *testing.T) { + var a interface{} = 1 + var b interface{} = &a + var c interface{} = &b + v, err := json.Marshal(c) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if string(v) != "1" { + t.Errorf("unexpected result: %v", string(v)) + } +} diff --git a/internal/encoder/opcode.go b/internal/encoder/opcode.go index b02ae351..05fc3ce0 100644 --- a/internal/encoder/opcode.go +++ b/internal/encoder/opcode.go @@ -363,7 +363,7 @@ func copyOpcode(code *Opcode) *Opcode { func setTotalLengthToInterfaceOp(code *Opcode) { for c := code; !c.IsEnd(); { - if c.Op == OpInterface { + if c.Op == OpInterface || c.Op == OpInterfacePtr { c.Length = uint32(code.TotalLength()) } c = c.IterNext() From 22be3b9a93e432bd30b92b68461a2c35356b9a83 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Fri, 22 Apr 2022 01:14:59 +0900 Subject: [PATCH 09/43] Update CHANGELOG --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9139b9b2..4c4a94fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# v0.9.7 - 2022/04/22 + +### Fix bugs + +#### Encoder + +* Add filtering process for encoding on slow path ( #355 ) +* Fix encoding of interface{} with pointer type ( #363 ) + +#### Decoder + +* Fix map key decoder that implements UnmarshalJSON ( #353 ) +* Fix decoding of []uint8 type ( #361 ) + +### New features + +* Add DebugWith option for encoder ( #356 ) + # v0.9.6 - 2022/03/22 ### Fix bugs From 6911114fb4efaa8438bce75ea0f9c2ebd59ba6e5 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Tue, 26 Apr 2022 01:40:44 +0900 Subject: [PATCH 10/43] fix: to care surrogate-pair on stringDecoder fix #364 --- decode_test.go | 13 +++++++++++++ internal/decoder/string.go | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/decode_test.go b/decode_test.go index 1d9f47c5..f7c40748 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3931,3 +3931,16 @@ func TestIssue359(t *testing.T) { t.Errorf("unexpected result: %v", string(v)) } } + +func TestIssue364(t *testing.T) { + var v struct { + Description string `json:"description"` + } + err := json.Unmarshal([]byte(`{"description":"\uD83D\uDE87 Toledo is a metro station"}`), &v) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if v.Description != "🚇 Toledo is a metro station" { + t.Errorf("unexpected result: %v", v.Description) + } +} diff --git a/internal/decoder/string.go b/internal/decoder/string.go index a55ac556..dc0a0108 100644 --- a/internal/decoder/string.go +++ b/internal/decoder/string.go @@ -386,6 +386,19 @@ func unescapeString(buf []byte) int { v3 := hexToInt[char(src, 4)] v4 := hexToInt[char(src, 5)] code := rune((v1 << 12) | (v2 << 8) | (v3 << 4) | v4) + if code >= 0xd800 && code < 0xdc00 && uintptr(unsafeAdd(src, 11)) < uintptr(end) { + if char(src, 6) == '\\' && char(src, 7) == 'u' { + v1 := hexToInt[char(src, 8)] + v2 := hexToInt[char(src, 9)] + v3 := hexToInt[char(src, 10)] + v4 := hexToInt[char(src, 11)] + lo := rune((v1 << 12) | (v2 << 8) | (v3 << 4) | v4) + if lo >= 0xdc00 && lo < 0xe000 { + code = (code-0xd800)<<10 | (lo - 0xdc00) + 0x10000 + src = unsafeAdd(src, 6) + } + } + } var b [utf8.UTFMax]byte n := utf8.EncodeRune(b[:], code) switch n { From af33c478461d9d790e46e65ba2fdc9214a42988a Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Tue, 26 Apr 2022 14:16:28 +0900 Subject: [PATCH 11/43] fix: determining embedded structs was wrong fix #362 --- decode_test.go | 15 +++++++++++++++ internal/decoder/compile.go | 9 +++++++++ internal/encoder/code.go | 16 ++++++++++++++-- internal/runtime/struct_field.go | 6 +++++- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/decode_test.go b/decode_test.go index f7c40748..93b7f410 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3944,3 +3944,18 @@ func TestIssue364(t *testing.T) { t.Errorf("unexpected result: %v", v.Description) } } + +func TestIssue362(t *testing.T) { + type AliasedPrimitive int + type Combiner struct { + SomeField int + AliasedPrimitive + } + originalCombiner := Combiner{AliasedPrimitive: 7} + b, err := json.Marshal(originalCombiner) + assertErr(t, err) + newCombiner := Combiner{} + err = json.Unmarshal(b, &newCombiner) + assertErr(t, err) + assertEq(t, "TestEmbeddedPrimitiveAlias", originalCombiner, newCombiner) +} diff --git a/internal/decoder/compile.go b/internal/decoder/compile.go index bf17d1b2..f13b43b8 100644 --- a/internal/decoder/compile.go +++ b/internal/decoder/compile.go @@ -394,6 +394,15 @@ func compileStruct(typ *runtime.Type, structName, fieldName string, structTypeTo allFields = append(allFields, fieldSet) } } + } else { + fieldSet := &structFieldSet{ + dec: dec, + offset: field.Offset, + isTaggedKey: tag.IsTaggedKey, + key: field.Name, + keyLen: int64(len(field.Name)), + } + allFields = append(allFields, fieldSet) } } else { if tag.IsString && isStringTagSupportedType(runtime.Type2RType(field.Type)) { diff --git a/internal/encoder/code.go b/internal/encoder/code.go index a00e38a9..a886480e 100644 --- a/internal/encoder/code.go +++ b/internal/encoder/code.go @@ -2,6 +2,7 @@ package encoder import ( "fmt" + "reflect" "unsafe" "github.com/goccy/go-json/internal/runtime" @@ -383,7 +384,7 @@ func (c *StructCode) Kind() CodeKind { } func (c *StructCode) lastFieldCode(field *StructFieldCode, firstField *Opcode) *Opcode { - if field.isAnonymous { + if isEmbeddedStruct(field) { return c.lastAnonymousFieldCode(firstField) } lastField := firstField @@ -436,7 +437,7 @@ func (c *StructCode) ToOpcode(ctx *compileContext) Opcodes { } if isEndField { endField := fieldCodes.Last() - if field.isAnonymous { + if isEmbeddedStruct(field) { firstField.End = endField lastField := c.lastAnonymousFieldCode(firstField) lastField.NextField = endField @@ -1003,3 +1004,14 @@ func convertPtrOp(code *Opcode) OpType { } return code.Op } + +func isEmbeddedStruct(field *StructFieldCode) bool { + if !field.isAnonymous { + return false + } + t := field.typ + if t.Kind() == reflect.Pointer { + t = t.Elem() + } + return t.Kind() == reflect.Struct +} diff --git a/internal/runtime/struct_field.go b/internal/runtime/struct_field.go index c321180a..eba42572 100644 --- a/internal/runtime/struct_field.go +++ b/internal/runtime/struct_field.go @@ -13,7 +13,11 @@ func getTag(field reflect.StructField) string { func IsIgnoredStructField(field reflect.StructField) bool { if field.PkgPath != "" { if field.Anonymous { - if !(field.Type.Kind() == reflect.Ptr && field.Type.Elem().Kind() == reflect.Struct) && field.Type.Kind() != reflect.Struct { + t := field.Type + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if !field.IsExported() && t.Kind() != reflect.Struct { return true } } else { From 944f8be027ed8fd0d47069ba945bd0c4671055a0 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Thu, 28 Apr 2022 20:24:46 +0900 Subject: [PATCH 12/43] chore: remove IsExported --- internal/runtime/struct_field.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/runtime/struct_field.go b/internal/runtime/struct_field.go index eba42572..baab0c59 100644 --- a/internal/runtime/struct_field.go +++ b/internal/runtime/struct_field.go @@ -17,7 +17,7 @@ func IsIgnoredStructField(field reflect.StructField) bool { if t.Kind() == reflect.Ptr { t = t.Elem() } - if !field.IsExported() && t.Kind() != reflect.Struct { + if t.Kind() != reflect.Struct { return true } } else { From 66f8b2629d0f998b10e01dd03ad49790ee3da825 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Thu, 28 Apr 2022 20:30:06 +0900 Subject: [PATCH 13/43] chore: use reflect.Ptr --- internal/encoder/code.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/encoder/code.go b/internal/encoder/code.go index a886480e..8d62a9cd 100644 --- a/internal/encoder/code.go +++ b/internal/encoder/code.go @@ -1010,7 +1010,7 @@ func isEmbeddedStruct(field *StructFieldCode) bool { return false } t := field.typ - if t.Kind() == reflect.Pointer { + if t.Kind() == reflect.Ptr { t = t.Elem() } return t.Kind() == reflect.Struct From 42805aa953ddf44901387a224da1afed7824bfe1 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Fri, 29 Apr 2022 17:16:25 +0900 Subject: [PATCH 14/43] fix: add escape sequence validation fix #335 --- decode_test.go | 9 +++++++++ internal/decoder/string.go | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/decode_test.go b/decode_test.go index 93b7f410..f5e2563a 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3959,3 +3959,12 @@ func TestIssue362(t *testing.T) { assertErr(t, err) assertEq(t, "TestEmbeddedPrimitiveAlias", originalCombiner, newCombiner) } + +func TestIssue335(t *testing.T) { + var v []string + in := []byte(`["\u","A"]`) + err := json.Unmarshal(in, &v) + if err == nil { + t.Errorf("unexpected success") + } +} diff --git a/internal/decoder/string.go b/internal/decoder/string.go index dc0a0108..cef6688b 100644 --- a/internal/decoder/string.go +++ b/internal/decoder/string.go @@ -2,6 +2,7 @@ package decoder import ( "bytes" + "fmt" "reflect" "unicode" "unicode/utf16" @@ -323,6 +324,12 @@ func (d *stringDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, err if cursor+5 >= buflen { return nil, 0, errors.ErrUnexpectedEndOfJSON("escaped string", cursor) } + for i := int64(1); i <= 4; i++ { + c := char(b, cursor+i) + if !(('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')) { + return nil, 0, errors.ErrSyntax(fmt.Sprintf("json: invalid character %c in \\u hexadecimal character escape", c), cursor+i) + } + } cursor += 5 default: return nil, 0, errors.ErrUnexpectedEndOfJSON("escaped string", cursor) From c07df9add625a24d8def6ae0ee606b5149d0ef89 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Tue, 3 May 2022 04:03:05 +0900 Subject: [PATCH 15/43] feat: improve performance on linkRecursiveCode fix #331 --- internal/encoder/compiler.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index 0eb3d90f..1be01aa3 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -885,29 +885,40 @@ func (c *Compiler) codeToOpcode(ctx *compileContext, typ *runtime.Type, code Cod } func (c *Compiler) linkRecursiveCode(ctx *compileContext) { + recursiveCodes := map[uintptr]*CompiledCode{} for _, recursive := range *ctx.recursiveCodes { typeptr := uintptr(unsafe.Pointer(recursive.Type)) codes := ctx.structTypeToCodes[typeptr] - compiled := recursive.Jmp - compiled.Code = copyOpcode(codes.First()) - code := compiled.Code - code.End.Next = newEndOp(&compileContext{}, recursive.Type) + if recursiveCode, ok := recursiveCodes[typeptr]; ok { + *recursive.Jmp = *recursiveCode + continue + } + + code := copyOpcode(codes.First()) code.Op = code.Op.PtrHeadToHead() + lastCode := newEndOp(&compileContext{}, recursive.Type) + lastCode.Op = OpRecursiveEnd - beforeLastCode := code.End - lastCode := beforeLastCode.Next + // OpRecursiveEnd must set before call TotalLength + code.End.Next = lastCode totalLength := code.TotalLength() + + // Idx, ElemIdx, Length must set after call TotalLength lastCode.Idx = uint32((totalLength + 1) * uintptrSize) lastCode.ElemIdx = lastCode.Idx + uintptrSize lastCode.Length = lastCode.Idx + 2*uintptrSize - code.End.Next.Op = OpRecursiveEnd // extend length to alloc slot for elemIdx + length curTotalLength := uintptr(recursive.TotalLength()) + 3 nextTotalLength := uintptr(totalLength) + 3 + + compiled := recursive.Jmp + compiled.Code = code compiled.CurLen = curTotalLength compiled.NextLen = nextTotalLength compiled.Linked = true + + recursiveCodes[typeptr] = compiled } } From 2ea7ab6e2481d5cb91a8ec72855db3e28345d0ce Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Wed, 4 May 2022 23:40:12 +0900 Subject: [PATCH 16/43] fix: wrong the detection method of nilable fix #339 --- encode_test.go | 18 ++++++++++++++++++ internal/encoder/compiler.go | 3 +++ 2 files changed, 21 insertions(+) diff --git a/encode_test.go b/encode_test.go index 92f43043..59bb08ff 100644 --- a/encode_test.go +++ b/encode_test.go @@ -9,6 +9,7 @@ import ( "fmt" "log" "math" + "math/big" "reflect" "regexp" "strconv" @@ -2388,3 +2389,20 @@ func TestIssue324(t *testing.T) { t.Fatalf("failed to encode. expected %q but got %q", expected, got) } } + +func TestIssue339(t *testing.T) { + type T1 struct { + *big.Int + } + type T2 struct { + T1 T1 `json:"T1"` + } + v := T2{T1{Int: big.NewInt(10000)}} + b, err := json.Marshal(&v) + assertErr(t, err) + got := string(b) + expected := `{"T1":10000}` + if got != expected { + t.Errorf("unexpected result: %v != %v", got, expected) + } +} diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index 1be01aa3..de7323c8 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -853,6 +853,9 @@ func (c *Compiler) implementsMarshalText(typ *runtime.Type) bool { } func (c *Compiler) isNilableType(typ *runtime.Type) bool { + if !runtime.IfaceIndir(typ) { + return true + } switch typ.Kind() { case reflect.Ptr: return true From 865b21589003a0b3921986f029f6d9e4f16e8d52 Mon Sep 17 00:00:00 2001 From: Matthew Topol Date: Thu, 5 May 2022 12:12:27 -0400 Subject: [PATCH 17/43] fix stream tokenizing respecting UseNumber --- internal/decoder/stream.go | 7 +++++-- stream_test.go | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/decoder/stream.go b/internal/decoder/stream.go index 332d47d8..6f337d77 100644 --- a/internal/decoder/stream.go +++ b/internal/decoder/stream.go @@ -138,8 +138,11 @@ func (s *Stream) Token() (interface{}, error) { s.cursor++ case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': bytes := floatBytes(s) - s := *(*string)(unsafe.Pointer(&bytes)) - f64, err := strconv.ParseFloat(s, 64) + str := *(*string)(unsafe.Pointer(&bytes)) + if s.UseNumber { + return json.Number(str), nil + } + f64, err := strconv.ParseFloat(str, 64) if err != nil { return nil, err } diff --git a/stream_test.go b/stream_test.go index a834e1ff..edb680cc 100644 --- a/stream_test.go +++ b/stream_test.go @@ -7,6 +7,7 @@ package json_test import ( "bytes" "compress/gzip" + "fmt" "io" "io/ioutil" "log" @@ -431,6 +432,16 @@ func TestDecodeInStream(t *testing.T) { } } +func TestDecodeStreamUseNumber(t *testing.T) { + dec := json.NewDecoder(strings.NewReader(`3.14`)) + dec.UseNumber() + v, err := dec.Token() + if err != nil { + t.Errorf("unexpected error: %#v", err) + } + assertEq(t, "json.Number", "json.Number", fmt.Sprintf("%T", v)) +} + // Test from golang.org/issue/11893 func TestHTTPDecoding(t *testing.T) { const raw = `{ "foo": "bar" }` From 27bd0f2aab2965c8940fd78d1f0c7ca5b91f4d85 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Thu, 30 Jun 2022 02:38:52 +0900 Subject: [PATCH 18/43] Update CHANGELOG --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c4a94fc..6e4b93f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# v0.9.8 - 2022/06/30 + +### Fix bugs + +* Fix decoding of surrogate-pair ( #365 ) +* Fix handling of embedded primitive type ( #366 ) +* Add validation of escape sequence for decoder ( #367 ) +* Fix stream tokenizing respecting UseNumber ( #369 ) +* Fix encoding when struct pointer type that implements Marshal JSON is embedded ( #375 ) + +### Improve performance + +* Improve performance of linkRecursiveCode ( #368 ) + # v0.9.7 - 2022/04/22 ### Fix bugs From c8d6da88dd5e417e57d7125bb805874f78496fcd Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Sun, 3 Jul 2022 06:05:26 +0900 Subject: [PATCH 19/43] fix: confusing nil in direct interface with typed nil fix #376 --- encode_test.go | 18 ++++++++++++++++++ internal/cmd/generator/vm.go.tmpl | 4 +++- internal/encoder/vm/vm.go | 4 +++- internal/encoder/vm_color/vm.go | 4 +++- internal/encoder/vm_color_indent/vm.go | 4 +++- internal/encoder/vm_indent/vm.go | 4 +++- 6 files changed, 33 insertions(+), 5 deletions(-) diff --git a/encode_test.go b/encode_test.go index 59bb08ff..1c0de3cc 100644 --- a/encode_test.go +++ b/encode_test.go @@ -2406,3 +2406,21 @@ func TestIssue339(t *testing.T) { t.Errorf("unexpected result: %v != %v", got, expected) } } + +func TestIssue376(t *testing.T) { + type Container struct { + V interface{} `json:"value"` + } + type MapOnly struct { + Map map[string]int64 `json:"map"` + } + b, err := json.Marshal(Container{MapOnly{}}) + if err != nil { + t.Fatal(err) + } + got := string(b) + expected := `{"value":{"map":null}}` + if got != expected { + t.Errorf("unexpected result: %v != %v", got, expected) + } +} diff --git a/internal/cmd/generator/vm.go.tmpl b/internal/cmd/generator/vm.go.tmpl index 91b11e1f..cb4db570 100644 --- a/internal/cmd/generator/vm.go.tmpl +++ b/internal/cmd/generator/vm.go.tmpl @@ -3,6 +3,7 @@ package vm import ( "math" + "reflect" "sort" "unsafe" @@ -193,7 +194,8 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if ifacePtr == nil && !isDirectedNil { b = appendNullComma(ctx, b) code = code.Next break diff --git a/internal/encoder/vm/vm.go b/internal/encoder/vm/vm.go index 91b11e1f..cb4db570 100644 --- a/internal/encoder/vm/vm.go +++ b/internal/encoder/vm/vm.go @@ -3,6 +3,7 @@ package vm import ( "math" + "reflect" "sort" "unsafe" @@ -193,7 +194,8 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if ifacePtr == nil && !isDirectedNil { b = appendNullComma(ctx, b) code = code.Next break diff --git a/internal/encoder/vm_color/vm.go b/internal/encoder/vm_color/vm.go index 5c6c52c3..e8a55974 100644 --- a/internal/encoder/vm_color/vm.go +++ b/internal/encoder/vm_color/vm.go @@ -3,6 +3,7 @@ package vm_color import ( "math" + "reflect" "sort" "unsafe" @@ -193,7 +194,8 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if ifacePtr == nil && !isDirectedNil { b = appendNullComma(ctx, b) code = code.Next break diff --git a/internal/encoder/vm_color_indent/vm.go b/internal/encoder/vm_color_indent/vm.go index 42dc11ca..79cfdd8d 100644 --- a/internal/encoder/vm_color_indent/vm.go +++ b/internal/encoder/vm_color_indent/vm.go @@ -3,6 +3,7 @@ package vm_color_indent import ( "math" + "reflect" "sort" "unsafe" @@ -193,7 +194,8 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if ifacePtr == nil && !isDirectedNil { b = appendNullComma(ctx, b) code = code.Next break diff --git a/internal/encoder/vm_indent/vm.go b/internal/encoder/vm_indent/vm.go index dfe0cc64..54514846 100644 --- a/internal/encoder/vm_indent/vm.go +++ b/internal/encoder/vm_indent/vm.go @@ -3,6 +3,7 @@ package vm_indent import ( "math" + "reflect" "sort" "unsafe" @@ -193,7 +194,8 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if ifacePtr == nil && !isDirectedNil { b = appendNullComma(ctx, b) code = code.Next break From 884b8dbf9ac90370b4df41d1b3cba565dc602078 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Sun, 3 Jul 2022 22:39:31 +0900 Subject: [PATCH 20/43] refactor: to check for IsDirectedNil only if ifacePtr == nil --- internal/cmd/generator/vm.go.tmpl | 12 +++++++----- internal/encoder/vm/vm.go | 12 +++++++----- internal/encoder/vm_color/vm.go | 12 +++++++----- internal/encoder/vm_color_indent/vm.go | 12 +++++++----- internal/encoder/vm_indent/vm.go | 12 +++++++----- 5 files changed, 35 insertions(+), 25 deletions(-) diff --git a/internal/cmd/generator/vm.go.tmpl b/internal/cmd/generator/vm.go.tmpl index cb4db570..645d20f9 100644 --- a/internal/cmd/generator/vm.go.tmpl +++ b/internal/cmd/generator/vm.go.tmpl @@ -194,11 +194,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) - if ifacePtr == nil && !isDirectedNil { - b = appendNullComma(ctx, b) - code = code.Next - break + if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) diff --git a/internal/encoder/vm/vm.go b/internal/encoder/vm/vm.go index cb4db570..645d20f9 100644 --- a/internal/encoder/vm/vm.go +++ b/internal/encoder/vm/vm.go @@ -194,11 +194,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) - if ifacePtr == nil && !isDirectedNil { - b = appendNullComma(ctx, b) - code = code.Next - break + if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) diff --git a/internal/encoder/vm_color/vm.go b/internal/encoder/vm_color/vm.go index e8a55974..a63e83e5 100644 --- a/internal/encoder/vm_color/vm.go +++ b/internal/encoder/vm_color/vm.go @@ -194,11 +194,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) - if ifacePtr == nil && !isDirectedNil { - b = appendNullComma(ctx, b) - code = code.Next - break + if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) diff --git a/internal/encoder/vm_color_indent/vm.go b/internal/encoder/vm_color_indent/vm.go index 79cfdd8d..3b4e22e5 100644 --- a/internal/encoder/vm_color_indent/vm.go +++ b/internal/encoder/vm_color_indent/vm.go @@ -194,11 +194,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) - if ifacePtr == nil && !isDirectedNil { - b = appendNullComma(ctx, b) - code = code.Next - break + if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) diff --git a/internal/encoder/vm_indent/vm.go b/internal/encoder/vm_indent/vm.go index 54514846..836c5c8a 100644 --- a/internal/encoder/vm_indent/vm.go +++ b/internal/encoder/vm_indent/vm.go @@ -194,11 +194,13 @@ func Run(ctx *encoder.RuntimeContext, b []byte, codeSet *encoder.OpcodeSet) ([]b ifacePtr = iface.ptr typ = iface.typ } - isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) - if ifacePtr == nil && !isDirectedNil { - b = appendNullComma(ctx, b) - code = code.Next - break + if ifacePtr == nil { + isDirectedNil := typ != nil && typ.Kind() == reflect.Struct && !runtime.IfaceIndir(typ) + if !isDirectedNil { + b = appendNullComma(ctx, b) + code = code.Next + break + } } ctx.KeepRefs = append(ctx.KeepRefs, up) ifaceCodeSet, err := encoder.CompileToGetCodeSet(ctx, uintptr(unsafe.Pointer(typ))) From f0e6a549f2f868b6ecbb5e05aa80b52d9baf9aee Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Mon, 4 Jul 2022 14:46:17 +0900 Subject: [PATCH 21/43] fix: support for embedding alias of primitive types fix #372 --- decode_test.go | 17 +++++++++++++++++ internal/decoder/compile.go | 9 +++++++++ 2 files changed, 26 insertions(+) diff --git a/decode_test.go b/decode_test.go index f5e2563a..c276a324 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3968,3 +3968,20 @@ func TestIssue335(t *testing.T) { t.Errorf("unexpected success") } } + +func TestIssue372(t *testing.T) { + type A int + type T struct { + _ int + *A + } + var v T + err := json.Unmarshal([]byte(`{"A":7}`), &v) + assertErr(t, err) + + got := *v.A + expected := A(7) + if got != expected { + t.Errorf("unexpected result: %v != %v", got, expected) + } +} diff --git a/internal/decoder/compile.go b/internal/decoder/compile.go index f13b43b8..48b02176 100644 --- a/internal/decoder/compile.go +++ b/internal/decoder/compile.go @@ -393,6 +393,15 @@ func compileStruct(typ *runtime.Type, structName, fieldName string, structTypeTo } allFields = append(allFields, fieldSet) } + } else { + fieldSet := &structFieldSet{ + dec: pdec, + offset: field.Offset, + isTaggedKey: tag.IsTaggedKey, + key: field.Name, + keyLen: int64(len(field.Name)), + } + allFields = append(allFields, fieldSet) } } else { fieldSet := &structFieldSet{ From 565e07e45c3fff959ae82cfeff814309fe185928 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Tue, 5 Jul 2022 01:15:08 +0900 Subject: [PATCH 22/43] fix: change isPtr to true on listElemCode fix #370 --- encode_test.go | 29 +++++++++++++++++++++++++++++ internal/encoder/compiler.go | 3 ++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/encode_test.go b/encode_test.go index 1c0de3cc..480ec78c 100644 --- a/encode_test.go +++ b/encode_test.go @@ -2424,3 +2424,32 @@ func TestIssue376(t *testing.T) { t.Errorf("unexpected result: %v != %v", got, expected) } } + +type Issue370 struct { + String string + Valid bool +} + +func (i *Issue370) MarshalJSON() ([]byte, error) { + if !i.Valid { + return json.Marshal(nil) + } + return json.Marshal(i.String) +} + +func TestIssue370(t *testing.T) { + v := []struct { + V Issue370 + }{ + {V: Issue370{String: "test", Valid: true}}, + } + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + got := string(b) + expected := `[{"V":"test"}]` + if got != expected { + t.Errorf("unexpected result: %v != %v", got, expected) + } +} diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index de7323c8..83eab572 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -487,7 +487,8 @@ func (c *Compiler) listElemCode(typ *runtime.Type) (Code, error) { case typ.Kind() == reflect.Map: return c.ptrCode(runtime.PtrTo(typ)) default: - code, err := c.typeToCodeWithPtr(typ, false) + // Strictly not isPtr == true, but reflect.ValueOf().Index() is canAddr, so set isPtr == true. + code, err := c.typeToCodeWithPtr(typ, true) if err != nil { return nil, err } From 8f5055b06ad27a023d7bcac7fc0f6ef03f2e8306 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Wed, 6 Jul 2022 16:56:11 +0900 Subject: [PATCH 23/43] fix: In decodeUnicode, the case that the expected buffer's state is not satisfied after reading. fix #374 --- encode_test.go | 15 +++++++++++++++ internal/decoder/string.go | 24 +++++++++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/encode_test.go b/encode_test.go index 1c0de3cc..f828dda8 100644 --- a/encode_test.go +++ b/encode_test.go @@ -7,12 +7,14 @@ import ( stdjson "encoding/json" "errors" "fmt" + "io" "log" "math" "math/big" "reflect" "regexp" "strconv" + "strings" "testing" "time" @@ -2424,3 +2426,16 @@ func TestIssue376(t *testing.T) { t.Errorf("unexpected result: %v != %v", got, expected) } } + +func TestIssue374(t *testing.T) { + r := io.MultiReader(strings.NewReader(strings.Repeat(" ", 505)+`"\u`), strings.NewReader(`0000"`)) + var v interface{} + if err := json.NewDecoder(r).Decode(&v); err != nil { + t.Fatal(err) + } + got := v.(string) + expected := "\u0000" + if got != expected { + t.Errorf("unexpected result: %q != %q", got, expected) + } +} diff --git a/internal/decoder/string.go b/internal/decoder/string.go index cef6688b..871ab3d7 100644 --- a/internal/decoder/string.go +++ b/internal/decoder/string.go @@ -95,24 +95,30 @@ func unicodeToRune(code []byte) rune { return r } +func readAtLeast(s *Stream, n int64, p *unsafe.Pointer) bool { + for s.cursor+n >= s.length { + if !s.read() { + return false + } + *p = s.bufptr() + } + return true +} + func decodeUnicodeRune(s *Stream, p unsafe.Pointer) (rune, int64, unsafe.Pointer, error) { const defaultOffset = 5 const surrogateOffset = 11 - if s.cursor+defaultOffset >= s.length { - if !s.read() { - return rune(0), 0, nil, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset()) - } - p = s.bufptr() + if !readAtLeast(s, defaultOffset, &p) { + return rune(0), 0, nil, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset()) } r := unicodeToRune(s.buf[s.cursor+1 : s.cursor+defaultOffset]) if utf16.IsSurrogate(r) { - if s.cursor+surrogateOffset >= s.length { - s.read() - p = s.bufptr() + if !readAtLeast(s, surrogateOffset, &p) { + return unicode.ReplacementChar, defaultOffset, p, nil } - if s.cursor+surrogateOffset >= s.length || s.buf[s.cursor+defaultOffset] != '\\' || s.buf[s.cursor+defaultOffset+1] != 'u' { + if s.buf[s.cursor+defaultOffset] != '\\' || s.buf[s.cursor+defaultOffset+1] != 'u' { return unicode.ReplacementChar, defaultOffset, p, nil } r2 := unicodeToRune(s.buf[s.cursor+defaultOffset+2 : s.cursor+surrogateOffset]) From 88aa13e3006fb5de4fa8c800778b4c671853e399 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Thu, 7 Jul 2022 14:52:28 +0900 Subject: [PATCH 24/43] Fix comment for #379 --- internal/encoder/compiler.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index 83eab572..64533f44 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -487,7 +487,9 @@ func (c *Compiler) listElemCode(typ *runtime.Type) (Code, error) { case typ.Kind() == reflect.Map: return c.ptrCode(runtime.PtrTo(typ)) default: - // Strictly not isPtr == true, but reflect.ValueOf().Index() is canAddr, so set isPtr == true. + // isPtr was originally used to indicate whether the type of top level is pointer. + // However, since the slice/array element is a specification that can get the pointer address, explicitly set isPtr to true. + // See here for related issues: https://github.com/goccy/go-json/issues/370 code, err := c.typeToCodeWithPtr(typ, true) if err != nil { return nil, err From 9a9f9adb055ce1cd8cf8251105633938f9f36875 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Sun, 10 Jul 2022 09:15:33 +0800 Subject: [PATCH 25/43] fix encoder and decoder cache slice edge case --- encode_test.go | 123 +++++++++++++++++++++++++++++++++++ internal/decoder/compile.go | 2 +- internal/encoder/compiler.go | 2 +- 3 files changed, 125 insertions(+), 2 deletions(-) diff --git a/encode_test.go b/encode_test.go index 4989866e..5b10d75f 100644 --- a/encode_test.go +++ b/encode_test.go @@ -2468,3 +2468,126 @@ func TestIssue374(t *testing.T) { t.Errorf("unexpected result: %q != %q", got, expected) } } + +func TestIssue381(t *testing.T) { + var v struct { + Field0 bool + Field1 bool + Field2 bool + Field3 bool + Field4 bool + Field5 bool + Field6 bool + Field7 bool + Field8 bool + Field9 bool + Field10 bool + Field11 bool + Field12 bool + Field13 bool + Field14 bool + Field15 bool + Field16 bool + Field17 bool + Field18 bool + Field19 bool + Field20 bool + Field21 bool + Field22 bool + Field23 bool + Field24 bool + Field25 bool + Field26 bool + Field27 bool + Field28 bool + Field29 bool + Field30 bool + Field31 bool + Field32 bool + Field33 bool + Field34 bool + Field35 bool + Field36 bool + Field37 bool + Field38 bool + Field39 bool + Field40 bool + Field41 bool + Field42 bool + Field43 bool + Field44 bool + Field45 bool + Field46 bool + Field47 bool + Field48 bool + Field49 bool + Field50 bool + Field51 bool + Field52 bool + Field53 bool + Field54 bool + Field55 bool + Field56 bool + Field57 bool + Field58 bool + Field59 bool + Field60 bool + Field61 bool + Field62 bool + Field63 bool + Field64 bool + Field65 bool + Field66 bool + Field67 bool + Field68 bool + Field69 bool + Field70 bool + Field71 bool + Field72 bool + Field73 bool + Field74 bool + Field75 bool + Field76 bool + Field77 bool + Field78 bool + Field79 bool + Field80 bool + Field81 bool + Field82 bool + Field83 bool + Field84 bool + Field85 bool + Field86 bool + Field87 bool + Field88 bool + Field89 bool + Field90 bool + Field91 bool + Field92 bool + Field93 bool + Field94 bool + Field95 bool + Field96 bool + Field97 bool + Field98 bool + Field99 bool + } + + // test encoder cache issue, not related to encoder + b, err := json.Marshal(v) + if err != nil { + t.Errorf("failed to marshal %s", err.Error()) + t.FailNow() + } + + std, err := stdjson.Marshal(v) + if err != nil { + t.Errorf("failed to marshal with encoding/json %s", err.Error()) + t.FailNow() + } + + if !bytes.Equal(std, b) { + t.Errorf("encoding result not equal to encoding/json") + t.FailNow() + } +} diff --git a/internal/decoder/compile.go b/internal/decoder/compile.go index 48b02176..fab64376 100644 --- a/internal/decoder/compile.go +++ b/internal/decoder/compile.go @@ -24,7 +24,7 @@ func init() { if typeAddr == nil { typeAddr = &runtime.TypeAddr{} } - cachedDecoder = make([]Decoder, typeAddr.AddrRange>>typeAddr.AddrShift) + cachedDecoder = make([]Decoder, typeAddr.AddrRange>>typeAddr.AddrShift+1) } func loadDecoderMap() map[uintptr]Decoder { diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index 64533f44..bf5e0f94 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -31,7 +31,7 @@ func init() { if typeAddr == nil { typeAddr = &runtime.TypeAddr{} } - cachedOpcodeSets = make([]*OpcodeSet, typeAddr.AddrRange>>typeAddr.AddrShift) + cachedOpcodeSets = make([]*OpcodeSet, typeAddr.AddrRange>>typeAddr.AddrShift+1) } func loadOpcodeMap() map[uintptr]*OpcodeSet { From 190c2e30bda23da7b7cf121ddd2809de8e3e687d Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Fri, 15 Jul 2022 16:59:13 +0900 Subject: [PATCH 26/43] Update CHANGELOG --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e4b93f1..e4b1d26f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# v0.9.9 - 2022/07/15 + +### Fix bugs + +* Fix encoding of directed interface with typed nil ( #377 ) +* Fix embedded primitive type encoding using alias ( #378 ) +* Fix slice/array type encoding with types implementing MarshalJSON ( #379 ) +* Fix unicode decoding when the expected buffer state is not met after reading ( #380 ) + # v0.9.8 - 2022/06/30 ### Fix bugs From 3eafdb612986f7fdf5dedbb537521def4341e3be Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Fri, 15 Jul 2022 19:12:47 +0900 Subject: [PATCH 27/43] Update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4b1d26f..20d13e97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# v0.9.10 - 2022/07/15 + +### Fix bugs + +* Fix boundary exception of type caching ( #382 ) + # v0.9.9 - 2022/07/15 ### Fix bugs From 70d6286ba8f88c6430355e3eae46414b2249f1bf Mon Sep 17 00:00:00 2001 From: KimHyeonwoo Date: Mon, 18 Jul 2022 19:41:08 +0900 Subject: [PATCH 28/43] fix cursor issue --- internal/decoder/stream.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/decoder/stream.go b/internal/decoder/stream.go index 6f337d77..8e21c568 100644 --- a/internal/decoder/stream.go +++ b/internal/decoder/stream.go @@ -281,6 +281,7 @@ func (s *Stream) skipObject(depth int64) error { s.cursor = cursor if s.read() { _, cursor, p = s.statForRetry() + cursor++ continue } return errors.ErrUnexpectedEndOfJSON("string of object", cursor) From 61705df0897d6fc787410aca5bf7b1ee9644f5e2 Mon Sep 17 00:00:00 2001 From: KimHyeonwoo Date: Mon, 18 Jul 2022 21:34:12 +0900 Subject: [PATCH 29/43] add test --- decode_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/decode_test.go b/decode_test.go index c276a324..8c8561b1 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3985,3 +3985,18 @@ func TestIssue372(t *testing.T) { t.Errorf("unexpected result: %v != %v", got, expected) } } + +type issue384 string + +func (t *issue384) UnmarshalJSON(b []byte) error { + return nil +} + +func TestIssue384(t *testing.T) { + in := "{\"data\":{\"text\": \"" + strings.Repeat("-", 492) + "\\\"\"}}\n" + dec := json.NewDecoder(strings.NewReader(in)) + var v issue384 + if err := dec.Decode(&v); err != nil { + t.Errorf("unexpected error: %v", err) + } +} From 3e25104a7cda232e682b2e88bc07830b13dd405c Mon Sep 17 00:00:00 2001 From: KimHyeonwoo Date: Mon, 18 Jul 2022 21:35:32 +0900 Subject: [PATCH 30/43] fix cursor issue for `skipArray`, `skipValue` --- internal/decoder/stream.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/decoder/stream.go b/internal/decoder/stream.go index 8e21c568..f1a58099 100644 --- a/internal/decoder/stream.go +++ b/internal/decoder/stream.go @@ -345,6 +345,7 @@ func (s *Stream) skipArray(depth int64) error { s.cursor = cursor if s.read() { _, cursor, p = s.statForRetry() + cursor++ continue } return errors.ErrUnexpectedEndOfJSON("string of object", cursor) @@ -403,6 +404,7 @@ func (s *Stream) skipValue(depth int64) error { s.cursor = cursor if s.read() { _, cursor, p = s.statForRetry() + cursor++ continue } return errors.ErrUnexpectedEndOfJSON("value of string", s.totalOffset()) From 229339ecd55154c1d3a0a637a95d178268ba2d36 Mon Sep 17 00:00:00 2001 From: KimHyeonwoo Date: Mon, 18 Jul 2022 22:03:42 +0900 Subject: [PATCH 31/43] add more test --- decode_test.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/decode_test.go b/decode_test.go index 8c8561b1..194c8229 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3986,17 +3986,22 @@ func TestIssue372(t *testing.T) { } } -type issue384 string +type issue384 struct{} func (t *issue384) UnmarshalJSON(b []byte) error { return nil } func TestIssue384(t *testing.T) { - in := "{\"data\":{\"text\": \"" + strings.Repeat("-", 492) + "\\\"\"}}\n" - dec := json.NewDecoder(strings.NewReader(in)) - var v issue384 - if err := dec.Decode(&v); err != nil { - t.Errorf("unexpected error: %v", err) + testcases := []string{ + "{\"data\":{\"text\": \"" + strings.Repeat("-", 492) + "\\\"\"}}\n", + "[\"" + strings.Repeat("-", 508) + "\\\"\"]", + } + for _, tc := range testcases { + dec := json.NewDecoder(strings.NewReader(tc)) + var v issue384 + if err := dec.Decode(&v); err != nil { + t.Errorf("unexpected error: %v", err) + } } } From f584919518468cd0891324d6a229bb24e64f64f1 Mon Sep 17 00:00:00 2001 From: KimHyeonwoo Date: Mon, 18 Jul 2022 22:14:48 +0900 Subject: [PATCH 32/43] fix testcase to more clear one --- decode_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decode_test.go b/decode_test.go index 194c8229..0f9485b1 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3994,7 +3994,7 @@ func (t *issue384) UnmarshalJSON(b []byte) error { func TestIssue384(t *testing.T) { testcases := []string{ - "{\"data\":{\"text\": \"" + strings.Repeat("-", 492) + "\\\"\"}}\n", + "{\"data\": \"" + strings.Repeat("-", 500) + "\\\"\"}", "[\"" + strings.Repeat("-", 508) + "\\\"\"]", } for _, tc := range testcases { From e5e8ed62c8c1a1aaaaaed73be21a31e8996365bf Mon Sep 17 00:00:00 2001 From: Tommy Hyeonwoo Kim Date: Tue, 19 Jul 2022 00:06:47 +0900 Subject: [PATCH 33/43] Update decode_test.go Co-authored-by: Sungyun Hur --- decode_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/decode_test.go b/decode_test.go index 0f9485b1..088367e5 100644 --- a/decode_test.go +++ b/decode_test.go @@ -3994,8 +3994,8 @@ func (t *issue384) UnmarshalJSON(b []byte) error { func TestIssue384(t *testing.T) { testcases := []string{ - "{\"data\": \"" + strings.Repeat("-", 500) + "\\\"\"}", - "[\"" + strings.Repeat("-", 508) + "\\\"\"]", + `{"data": "` + strings.Repeat("-", 500) + `\""}`, + `["` + strings.Repeat("-", 508) + `\""]`, } for _, tc := range testcases { dec := json.NewDecoder(strings.NewReader(tc)) From 95a32fc03836e0693db6688fd242fbfeff147ac0 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Fri, 29 Jul 2022 13:56:58 +0900 Subject: [PATCH 34/43] fix: forgot to update p after read fix #386 --- encode_test.go | 14 ++++++++++++++ internal/decoder/string.go | 1 + 2 files changed, 15 insertions(+) diff --git a/encode_test.go b/encode_test.go index 5b10d75f..b2cbceb7 100644 --- a/encode_test.go +++ b/encode_test.go @@ -2591,3 +2591,17 @@ func TestIssue381(t *testing.T) { t.FailNow() } } + +func TestIssue386(t *testing.T) { + raw := `{"date": null, "platform": "\u6f2b\u753b", "images": {"small": "https://lain.bgm.tv/pic/cover/s/d2/a1/80048_jp.jpg", "grid": "https://lain.bgm.tv/pic/cover/g/d2/a1/80048_jp.jpg", "large": "https://lain.bgm.tv/pic/cover/l/d2/a1/80048_jp.jpg", "medium": "https://lain.bgm.tv/pic/cover/m/d2/a1/80048_jp.jpg", "common": "https://lain.bgm.tv/pic/cover/c/d2/a1/80048_jp.jpg"}, "summary": "\u5929\u624d\u8a2d\u8a08\u58eb\u30fb\u5929\u5bae\uff08\u3042\u307e\u307f\u3084\uff09\u3092\u62b1\u3048\u308b\u6751\u96e8\u7dcf\u5408\u4f01\u753b\u306f\u3001\u771f\u6c34\u5efa\u8a2d\u3068\u63d0\u643a\u3057\u3066\u300c\u3055\u304d\u305f\u307e\u30a2\u30ea\u30fc\u30ca\u300d\u306e\u30b3\u30f3\u30da\u306b\u512a\u52dd\u3059\u308b\u3053\u3068\u306b\u8ced\u3051\u3066\u3044\u305f\u3002\u3057\u304b\u3057\u3001\u73fe\u77e5\u4e8b\u306e\u6d25\u5730\u7530\uff08\u3064\u3061\u3060\uff09\u306f\u5927\u65e5\u5efa\u8a2d\u306b\u512a\u52dd\u3055\u305b\u3088\u3046\u3068\u6697\u8e8d\u3059\u308b\u3002\u305d\u308c\u306f\u73fe\u77e5\u4e8b\u306e\u6d25\u5730\u7530\u3068\u526f\u77e5\u4e8b\u306e\u592a\u7530\uff08\u304a\u304a\u305f\uff09\u306e\u653f\u6cbb\u751f\u547d\u3092\u5de6\u53f3\u3059\u308b\u4e89\u3044\u3068\u306a\u308a\u2026\u2026!?\u3000\u305d\u3057\u3066\u516c\u5171\u4e8b\u696d\u306b\u6e26\u5dfb\u304f\u6df1\u3044\u95c7\u306b\u842c\u7530\u9280\u6b21\u90ce\uff08\u307e\u3093\u3060\u30fb\u304e\u3093\u3058\u308d\u3046\uff09\u306f\u2026\u2026!?", "name": "\u30df\u30ca\u30df\u306e\u5e1d\u738b (48)"}` + var a struct { + Date *string `json:"date"` + Platform *string `json:"platform"` + Summary string `json:"summary"` + Name string `json:"name"` + } + err := json.NewDecoder(strings.NewReader(raw)).Decode(&a) + if err != nil { + t.Error(err) + } +} diff --git a/internal/decoder/string.go b/internal/decoder/string.go index 871ab3d7..d07ad710 100644 --- a/internal/decoder/string.go +++ b/internal/decoder/string.go @@ -171,6 +171,7 @@ RETRY: if !s.read() { return nil, errors.ErrInvalidCharacter(s.char(), "escaped string", s.totalOffset()) } + p = s.bufptr() goto RETRY default: return nil, errors.ErrUnexpectedEndOfJSON("string", s.totalOffset()) From f83142d838f231e825c02e0d1c6b0b8ccdeff216 Mon Sep 17 00:00:00 2001 From: KimHyeonwoo Date: Tue, 2 Aug 2022 12:07:48 +0900 Subject: [PATCH 35/43] replace statForRetry with stat (review reflected) --- internal/decoder/stream.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/internal/decoder/stream.go b/internal/decoder/stream.go index f1a58099..a383f725 100644 --- a/internal/decoder/stream.go +++ b/internal/decoder/stream.go @@ -280,8 +280,7 @@ func (s *Stream) skipObject(depth int64) error { if char(p, cursor) == nul { s.cursor = cursor if s.read() { - _, cursor, p = s.statForRetry() - cursor++ + _, cursor, p = s.stat() continue } return errors.ErrUnexpectedEndOfJSON("string of object", cursor) @@ -344,8 +343,7 @@ func (s *Stream) skipArray(depth int64) error { if char(p, cursor) == nul { s.cursor = cursor if s.read() { - _, cursor, p = s.statForRetry() - cursor++ + _, cursor, p = s.stat() continue } return errors.ErrUnexpectedEndOfJSON("string of object", cursor) @@ -403,8 +401,7 @@ func (s *Stream) skipValue(depth int64) error { if char(p, cursor) == nul { s.cursor = cursor if s.read() { - _, cursor, p = s.statForRetry() - cursor++ + _, cursor, p = s.stat() continue } return errors.ErrUnexpectedEndOfJSON("value of string", s.totalOffset()) From 30713917a8e5740aa161ddd53f1be7fb74764533 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Thu, 18 Aug 2022 12:15:19 +0900 Subject: [PATCH 36/43] Update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20d13e97..d63009fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# v0.9.11 - 2022/08/18 + +### Fix bugs + +* Fix unexpected behavior when buffer ends with backslash ( #383 ) +* Fix stream decoding of escaped character ( #387 ) + # v0.9.10 - 2022/07/15 ### Fix bugs From 41ad89fe02230d0fc32d4698877df9d0670e903f Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Thu, 18 Aug 2022 12:48:27 +0900 Subject: [PATCH 37/43] Fix golangci-lint action --- .github/workflows/lint.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 684cbc76..98d20f23 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: golangci/golangci-lint-action@v2 + - uses: golangci/golangci-lint-action@v3 with: version: v1.45.2 args: --timeout=5m diff --git a/Makefile b/Makefile index 363563ab..7a10530b 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ golangci-lint: | $(BIN_DIR) GOLANGCI_LINT_TMP_DIR=$$(mktemp -d); \ cd $$GOLANGCI_LINT_TMP_DIR; \ go mod init tmp; \ - GOBIN=$(BIN_DIR) go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.36.0; \ + GOBIN=$(BIN_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.48.0; \ rm -rf $$GOLANGCI_LINT_TMP_DIR; \ } From 705f51716bfc706f41209b07164a3882b6be2b96 Mon Sep 17 00:00:00 2001 From: brongineers Date: Sun, 13 Nov 2022 21:05:50 +0300 Subject: [PATCH 38/43] fix custom marshal for map key --- encode_test.go | 15 +++++++++++++++ internal/encoder/compiler.go | 2 -- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/encode_test.go b/encode_test.go index b2cbceb7..c7afadb7 100644 --- a/encode_test.go +++ b/encode_test.go @@ -2605,3 +2605,18 @@ func TestIssue386(t *testing.T) { t.Error(err) } } + +type customMapKey string + +func (b customMapKey) MarshalJSON() ([]byte, error) { + return []byte("[]"), nil +} + +func TestCustomMarshalForMapKey(t *testing.T) { + m := map[customMapKey]string{customMapKey("skipcustom"): "test"} + expected, err := stdjson.Marshal(m) + assertErr(t, err) + got, err := json.Marshal(m) + assertErr(t, err) + assertEq(t, "custom map key", string(expected), string(got)) +} diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index bf5e0f94..3b3ff3fd 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -506,8 +506,6 @@ func (c *Compiler) listElemCode(typ *runtime.Type) (Code, error) { func (c *Compiler) mapKeyCode(typ *runtime.Type) (Code, error) { switch { - case c.implementsMarshalJSON(typ): - return c.marshalJSONCode(typ) case c.implementsMarshalText(typ): return c.marshalTextCode(typ) } From 6bca98964369e64b3157ae166c54af60834ef1eb Mon Sep 17 00:00:00 2001 From: brongineers Date: Mon, 14 Nov 2022 22:11:00 +0300 Subject: [PATCH 39/43] test key map MarshalText with std --- encode_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/encode_test.go b/encode_test.go index c7afadb7..63740f35 100644 --- a/encode_test.go +++ b/encode_test.go @@ -1022,12 +1022,13 @@ func (u *unmarshalerText) UnmarshalText(b []byte) error { } func TestTextMarshalerMapKeysAreSorted(t *testing.T) { - b, err := json.Marshal(map[unmarshalerText]int{ + data := map[unmarshalerText]int{ {"x", "y"}: 1, {"y", "x"}: 2, {"a", "z"}: 3, {"z", "a"}: 4, - }) + } + b, err := json.Marshal(data) if err != nil { t.Fatalf("Failed to Marshal text.Marshaler: %v", err) } @@ -1035,6 +1036,14 @@ func TestTextMarshalerMapKeysAreSorted(t *testing.T) { if string(b) != want { t.Errorf("Marshal map with text.Marshaler keys: got %#q, want %#q", b, want) } + + b, err = stdjson.Marshal(data) + if err != nil { + t.Fatalf("Failed to std Marshal text.Marshaler: %v", err) + } + if string(b) != want { + t.Errorf("std Marshal map with text.Marshaler keys: got %#q, want %#q", b, want) + } } // https://golang.org/issue/33675 From 2163995ea5531a0c0b67034b6cbadda7a0e616e9 Mon Sep 17 00:00:00 2001 From: Asaf Shitrit <62242636+asaf-shitrit@users.noreply.github.com> Date: Tue, 15 Nov 2022 14:10:06 +0200 Subject: [PATCH 40/43] Typo fix in README.md file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A minor typo in the sentence "Therefore, the arguments for `Marshal` and `Unmarshal` are always escape to the heap." The word "escape" should be "escaped" 🙂 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 56862377..7bacc54f 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ func Marshal(v interface{}) ([]byte, error) { `json.Marshal` and `json.Unmarshal` receive `interface{}` value and they perform type determination dynamically to process. In normal case, you need to use the `reflect` library to determine the type dynamically, but since `reflect.Type` is defined as `interface`, when you call the method of `reflect.Type`, The reflect's argument is escaped. -Therefore, the arguments for `Marshal` and `Unmarshal` are always escape to the heap. +Therefore, the arguments for `Marshal` and `Unmarshal` are always escaped to the heap. However, `go-json` can use the feature of `reflect.Type` while avoiding escaping. `reflect.Type` is defined as `interface`, but in reality `reflect.Type` is implemented only by the structure `rtype` defined in the `reflect` package. From 781a0b3e85b93ffac70ee8c9cf271a84c98eb85b Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 29 Nov 2022 03:55:56 +0900 Subject: [PATCH 41/43] Support JSON Path --- Makefile | 2 +- benchmarks/path_test.go | 24 + decode.go | 28 ++ error.go | 2 + internal/decoder/anonymous_field.go | 4 + internal/decoder/array.go | 5 + internal/decoder/assign.go | 438 +++++++++++++++++++ internal/decoder/bool.go | 5 + internal/decoder/bytes.go | 5 + internal/decoder/float.go | 12 + internal/decoder/func.go | 5 + internal/decoder/int.go | 4 + internal/decoder/interface.go | 70 +++ internal/decoder/invalid.go | 10 + internal/decoder/map.go | 92 ++++ internal/decoder/number.go | 11 + internal/decoder/option.go | 2 + internal/decoder/path.go | 653 ++++++++++++++++++++++++++++ internal/decoder/ptr.go | 9 + internal/decoder/slice.go | 77 ++++ internal/decoder/string.go | 11 + internal/decoder/struct.go | 4 + internal/decoder/type.go | 1 + internal/decoder/uint.go | 4 + internal/decoder/unmarshal_json.go | 5 + internal/decoder/unmarshal_text.go | 5 + internal/decoder/wrapped_string.go | 5 + internal/errors/error.go | 19 + path.go | 73 ++++ path_test.go | 234 ++++++++++ 30 files changed, 1818 insertions(+), 1 deletion(-) create mode 100644 benchmarks/path_test.go create mode 100644 internal/decoder/assign.go create mode 100644 internal/decoder/path.go create mode 100644 path.go create mode 100644 path_test.go diff --git a/Makefile b/Makefile index 7a10530b..5bbfc4c9 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ cover-html: cover .PHONY: lint lint: golangci-lint - golangci-lint run + $(BIN_DIR)/golangci-lint run golangci-lint: | $(BIN_DIR) @{ \ diff --git a/benchmarks/path_test.go b/benchmarks/path_test.go new file mode 100644 index 00000000..3224b657 --- /dev/null +++ b/benchmarks/path_test.go @@ -0,0 +1,24 @@ +package benchmark + +import ( + "testing" + + gojson "github.com/goccy/go-json" +) + +func Benchmark_Decode_SmallStruct_UnmarshalPath_GoJson(b *testing.B) { + path, err := gojson.CreatePath("$.st") + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var v int + if err := path.Unmarshal(SmallFixture, &v); err != nil { + b.Fatal(err) + } + if v != 1 { + b.Fatal("failed to unmarshal path") + } + } +} diff --git a/decode.go b/decode.go index d99749d0..b1512cec 100644 --- a/decode.go +++ b/decode.go @@ -83,6 +83,34 @@ func unmarshalContext(ctx context.Context, data []byte, v interface{}, optFuncs return validateEndBuf(src, cursor) } +var ( + pathDecoder = decoder.NewPathDecoder() +) + +func extractFromPath(path *Path, data []byte, optFuncs ...DecodeOptionFunc) ([][]byte, error) { + src := make([]byte, len(data)+1) // append nul byte to the end + copy(src, data) + + ctx := decoder.TakeRuntimeContext() + ctx.Buf = src + ctx.Option.Flags = 0 + ctx.Option.Flags |= decoder.PathOption + ctx.Option.Path = path.path + for _, optFunc := range optFuncs { + optFunc(ctx.Option) + } + paths, cursor, err := pathDecoder.DecodePath(ctx, 0, 0) + if err != nil { + decoder.ReleaseRuntimeContext(ctx) + return nil, err + } + decoder.ReleaseRuntimeContext(ctx) + if err := validateEndBuf(src, cursor); err != nil { + return nil, err + } + return paths, nil +} + func unmarshalNoEscape(data []byte, v interface{}, optFuncs ...DecodeOptionFunc) error { src := make([]byte, len(data)+1) // append nul byte to the end copy(src, data) diff --git a/error.go b/error.go index 94c1339a..5b2dcee5 100644 --- a/error.go +++ b/error.go @@ -37,3 +37,5 @@ type UnmarshalTypeError = errors.UnmarshalTypeError type UnsupportedTypeError = errors.UnsupportedTypeError type UnsupportedValueError = errors.UnsupportedValueError + +type PathError = errors.PathError diff --git a/internal/decoder/anonymous_field.go b/internal/decoder/anonymous_field.go index 030cb7a9..b6876cf0 100644 --- a/internal/decoder/anonymous_field.go +++ b/internal/decoder/anonymous_field.go @@ -35,3 +35,7 @@ func (d *anonymousFieldDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p = *(*unsafe.Pointer)(p) return d.dec.Decode(ctx, cursor, depth, unsafe.Pointer(uintptr(p)+d.offset)) } + +func (d *anonymousFieldDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return d.dec.DecodePath(ctx, cursor, depth) +} diff --git a/internal/decoder/array.go b/internal/decoder/array.go index 21f1fd58..8ef91cfa 100644 --- a/internal/decoder/array.go +++ b/internal/decoder/array.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "unsafe" "github.com/goccy/go-json/internal/errors" @@ -167,3 +168,7 @@ func (d *arrayDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe } } } + +func (d *arrayDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: array decoder does not support decode path") +} diff --git a/internal/decoder/assign.go b/internal/decoder/assign.go new file mode 100644 index 00000000..c53e6ad9 --- /dev/null +++ b/internal/decoder/assign.go @@ -0,0 +1,438 @@ +package decoder + +import ( + "fmt" + "reflect" + "strconv" +) + +var ( + nilValue = reflect.ValueOf(nil) +) + +func AssignValue(src, dst reflect.Value) error { + if dst.Type().Kind() != reflect.Ptr { + return fmt.Errorf("invalid dst type. required pointer type: %T", dst.Type()) + } + casted, err := castValue(dst.Elem().Type(), src) + if err != nil { + return err + } + dst.Elem().Set(casted) + return nil +} + +func castValue(t reflect.Type, v reflect.Value) (reflect.Value, error) { + switch t.Kind() { + case reflect.Int: + vv, err := castInt(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(int(vv.Int())), nil + case reflect.Int8: + vv, err := castInt(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(int8(vv.Int())), nil + case reflect.Int16: + vv, err := castInt(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(int16(vv.Int())), nil + case reflect.Int32: + vv, err := castInt(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(int32(vv.Int())), nil + case reflect.Int64: + return castInt(v) + case reflect.Uint: + vv, err := castUint(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(uint(vv.Uint())), nil + case reflect.Uint8: + vv, err := castUint(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(uint8(vv.Uint())), nil + case reflect.Uint16: + vv, err := castUint(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(uint16(vv.Uint())), nil + case reflect.Uint32: + vv, err := castUint(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(uint32(vv.Uint())), nil + case reflect.Uint64: + return castUint(v) + case reflect.Uintptr: + vv, err := castUint(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(uintptr(vv.Uint())), nil + case reflect.String: + return castString(v) + case reflect.Bool: + return castBool(v) + case reflect.Float32: + vv, err := castFloat(v) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(float32(vv.Float())), nil + case reflect.Float64: + return castFloat(v) + case reflect.Array: + return castArray(t, v) + case reflect.Slice: + return castSlice(t, v) + case reflect.Map: + return castMap(t, v) + case reflect.Struct: + return castStruct(t, v) + } + return v, nil +} + +func castInt(v reflect.Value) (reflect.Value, error) { + switch v.Type().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v, nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return reflect.ValueOf(int64(v.Uint())), nil + case reflect.String: + i64, err := strconv.ParseInt(v.String(), 10, 64) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(i64), nil + case reflect.Bool: + if v.Bool() { + return reflect.ValueOf(int64(1)), nil + } + return reflect.ValueOf(int64(0)), nil + case reflect.Float32, reflect.Float64: + return reflect.ValueOf(int64(v.Float())), nil + case reflect.Array: + if v.Len() > 0 { + return castInt(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to int64 from empty array") + case reflect.Slice: + if v.Len() > 0 { + return castInt(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to int64 from empty slice") + case reflect.Interface: + return castInt(reflect.ValueOf(v.Interface())) + case reflect.Map: + return nilValue, fmt.Errorf("failed to cast to int64 from map") + case reflect.Struct: + return nilValue, fmt.Errorf("failed to cast to int64 from struct") + case reflect.Ptr: + return castInt(v.Elem()) + } + return nilValue, fmt.Errorf("failed to cast to int64 from %s", v.Type().Kind()) +} + +func castUint(v reflect.Value) (reflect.Value, error) { + switch v.Type().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return reflect.ValueOf(uint64(v.Int())), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v, nil + case reflect.String: + u64, err := strconv.ParseUint(v.String(), 10, 64) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(u64), nil + case reflect.Bool: + if v.Bool() { + return reflect.ValueOf(uint64(1)), nil + } + return reflect.ValueOf(uint64(0)), nil + case reflect.Float32, reflect.Float64: + return reflect.ValueOf(uint64(v.Float())), nil + case reflect.Array: + if v.Len() > 0 { + return castUint(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to uint64 from empty array") + case reflect.Slice: + if v.Len() > 0 { + return castUint(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to uint64 from empty slice") + case reflect.Interface: + return castUint(reflect.ValueOf(v.Interface())) + case reflect.Map: + return nilValue, fmt.Errorf("failed to cast to uint64 from map") + case reflect.Struct: + return nilValue, fmt.Errorf("failed to cast to uint64 from struct") + case reflect.Ptr: + return castUint(v.Elem()) + } + return nilValue, fmt.Errorf("failed to cast to uint64 from %s", v.Type().Kind()) +} + +func castString(v reflect.Value) (reflect.Value, error) { + switch v.Type().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return reflect.ValueOf(fmt.Sprint(v.Int())), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return reflect.ValueOf(fmt.Sprint(v.Uint())), nil + case reflect.String: + return v, nil + case reflect.Bool: + if v.Bool() { + return reflect.ValueOf("true"), nil + } + return reflect.ValueOf("false"), nil + case reflect.Float32, reflect.Float64: + return reflect.ValueOf(fmt.Sprint(v.Float())), nil + case reflect.Array: + if v.Len() > 0 { + return castString(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to string from empty array") + case reflect.Slice: + if v.Len() > 0 { + return castString(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to string from empty slice") + case reflect.Interface: + return castString(reflect.ValueOf(v.Interface())) + case reflect.Map: + return nilValue, fmt.Errorf("failed to cast to string from map") + case reflect.Struct: + return nilValue, fmt.Errorf("failed to cast to string from struct") + case reflect.Ptr: + return castString(v.Elem()) + } + return nilValue, fmt.Errorf("failed to cast to string from %s", v.Type().Kind()) +} + +func castBool(v reflect.Value) (reflect.Value, error) { + switch v.Type().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch v.Int() { + case 0: + return reflect.ValueOf(false), nil + case 1: + return reflect.ValueOf(true), nil + } + return nilValue, fmt.Errorf("failed to cast to bool from %d", v.Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch v.Uint() { + case 0: + return reflect.ValueOf(false), nil + case 1: + return reflect.ValueOf(true), nil + } + return nilValue, fmt.Errorf("failed to cast to bool from %d", v.Uint()) + case reflect.String: + b, err := strconv.ParseBool(v.String()) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(b), nil + case reflect.Bool: + return v, nil + case reflect.Float32, reflect.Float64: + switch v.Float() { + case 0: + return reflect.ValueOf(false), nil + case 1: + return reflect.ValueOf(true), nil + } + return nilValue, fmt.Errorf("failed to cast to bool from %f", v.Float()) + case reflect.Array: + if v.Len() > 0 { + return castBool(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to string from empty array") + case reflect.Slice: + if v.Len() > 0 { + return castBool(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to string from empty slice") + case reflect.Interface: + return castBool(reflect.ValueOf(v.Interface())) + case reflect.Map: + return nilValue, fmt.Errorf("failed to cast to string from map") + case reflect.Struct: + return nilValue, fmt.Errorf("failed to cast to string from struct") + case reflect.Ptr: + return castBool(v.Elem()) + } + return nilValue, fmt.Errorf("failed to cast to bool from %s", v.Type().Kind()) +} + +func castFloat(v reflect.Value) (reflect.Value, error) { + switch v.Type().Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return reflect.ValueOf(float64(v.Int())), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return reflect.ValueOf(float64(v.Uint())), nil + case reflect.String: + f64, err := strconv.ParseFloat(v.String(), 64) + if err != nil { + return nilValue, err + } + return reflect.ValueOf(f64), nil + case reflect.Bool: + if v.Bool() { + return reflect.ValueOf(float64(1)), nil + } + return reflect.ValueOf(float64(0)), nil + case reflect.Float32, reflect.Float64: + return v, nil + case reflect.Array: + if v.Len() > 0 { + return castFloat(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to float64 from empty array") + case reflect.Slice: + if v.Len() > 0 { + return castFloat(v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to float64 from empty slice") + case reflect.Interface: + return castFloat(reflect.ValueOf(v.Interface())) + case reflect.Map: + return nilValue, fmt.Errorf("failed to cast to float64 from map") + case reflect.Struct: + return nilValue, fmt.Errorf("failed to cast to float64 from struct") + case reflect.Ptr: + return castFloat(v.Elem()) + } + return nilValue, fmt.Errorf("failed to cast to float64 from %s", v.Type().Kind()) +} + +func castArray(t reflect.Type, v reflect.Value) (reflect.Value, error) { + kind := v.Type().Kind() + if kind == reflect.Interface { + return castArray(t, reflect.ValueOf(v.Interface())) + } + if kind != reflect.Slice && kind != reflect.Array { + return nilValue, fmt.Errorf("failed to cast to array from %s", kind) + } + if t.Elem() == v.Type().Elem() { + return v, nil + } + if t.Len() != v.Len() { + return nilValue, fmt.Errorf("failed to cast [%d]array from slice of %d length", t.Len(), v.Len()) + } + ret := reflect.New(t).Elem() + for i := 0; i < v.Len(); i++ { + vv, err := castValue(t.Elem(), v.Index(i)) + if err != nil { + return nilValue, err + } + ret.Index(i).Set(vv) + } + return ret, nil +} + +func castSlice(t reflect.Type, v reflect.Value) (reflect.Value, error) { + kind := v.Type().Kind() + if kind == reflect.Interface { + return castSlice(t, reflect.ValueOf(v.Interface())) + } + if kind != reflect.Slice && kind != reflect.Array { + return nilValue, fmt.Errorf("failed to cast to slice from %s", kind) + } + if t.Elem() == v.Type().Elem() { + return v, nil + } + ret := reflect.MakeSlice(t, v.Len(), v.Len()) + for i := 0; i < v.Len(); i++ { + vv, err := castValue(t.Elem(), v.Index(i)) + if err != nil { + return nilValue, err + } + ret.Index(i).Set(vv) + } + return ret, nil +} + +func castMap(t reflect.Type, v reflect.Value) (reflect.Value, error) { + ret := reflect.MakeMap(t) + switch v.Type().Kind() { + case reflect.Map: + iter := v.MapRange() + for iter.Next() { + key, err := castValue(t.Key(), iter.Key()) + if err != nil { + return nilValue, err + } + value, err := castValue(t.Elem(), iter.Value()) + if err != nil { + return nilValue, err + } + ret.SetMapIndex(key, value) + } + return ret, nil + case reflect.Interface: + return castMap(t, reflect.ValueOf(v.Interface())) + case reflect.Slice: + if v.Len() > 0 { + return castMap(t, v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to map from empty slice") + } + return nilValue, fmt.Errorf("failed to cast to map from %s", v.Type().Kind()) +} + +func castStruct(t reflect.Type, v reflect.Value) (reflect.Value, error) { + ret := reflect.New(t).Elem() + switch v.Type().Kind() { + case reflect.Map: + iter := v.MapRange() + for iter.Next() { + key := iter.Key() + k, err := castString(key) + if err != nil { + return nilValue, err + } + fieldName := k.String() + field, ok := t.FieldByName(fieldName) + if ok { + value, err := castValue(field.Type, iter.Value()) + if err != nil { + return nilValue, err + } + ret.FieldByName(fieldName).Set(value) + } + } + return ret, nil + case reflect.Struct: + for i := 0; i < v.Type().NumField(); i++ { + name := v.Type().Field(i).Name + ret.FieldByName(name).Set(v.FieldByName(name)) + } + return ret, nil + case reflect.Interface: + return castStruct(t, reflect.ValueOf(v.Interface())) + case reflect.Slice: + if v.Len() > 0 { + return castStruct(t, v.Index(0)) + } + return nilValue, fmt.Errorf("failed to cast to struct from empty slice") + default: + return nilValue, fmt.Errorf("failed to cast to struct from %s", v.Type().Kind()) + } +} diff --git a/internal/decoder/bool.go b/internal/decoder/bool.go index 455042a5..ba6cf5bc 100644 --- a/internal/decoder/bool.go +++ b/internal/decoder/bool.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "unsafe" "github.com/goccy/go-json/internal/errors" @@ -76,3 +77,7 @@ func (d *boolDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe. } return 0, errors.ErrUnexpectedEndOfJSON("bool", cursor) } + +func (d *boolDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: bool decoder does not support decode path") +} diff --git a/internal/decoder/bytes.go b/internal/decoder/bytes.go index 92c7dcf6..939bf432 100644 --- a/internal/decoder/bytes.go +++ b/internal/decoder/bytes.go @@ -2,6 +2,7 @@ package decoder import ( "encoding/base64" + "fmt" "unsafe" "github.com/goccy/go-json/internal/errors" @@ -78,6 +79,10 @@ func (d *bytesDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe return cursor, nil } +func (d *bytesDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: []byte decoder does not support decode path") +} + func (d *bytesDecoder) decodeStreamBinary(s *Stream, depth int64, p unsafe.Pointer) ([]byte, error) { c := s.skipWhiteSpace() if c == '[' { diff --git a/internal/decoder/float.go b/internal/decoder/float.go index dfb7168d..9b2eb8b3 100644 --- a/internal/decoder/float.go +++ b/internal/decoder/float.go @@ -156,3 +156,15 @@ func (d *floatDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe d.op(p, f64) return cursor, nil } + +func (d *floatDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + buf := ctx.Buf + bytes, c, err := d.decodeByte(buf, cursor) + if err != nil { + return nil, 0, err + } + if bytes == nil { + return [][]byte{nullbytes}, c, nil + } + return [][]byte{bytes}, c, nil +} diff --git a/internal/decoder/func.go b/internal/decoder/func.go index ee356371..4cc12ca8 100644 --- a/internal/decoder/func.go +++ b/internal/decoder/func.go @@ -2,6 +2,7 @@ package decoder import ( "bytes" + "fmt" "unsafe" "github.com/goccy/go-json/internal/errors" @@ -139,3 +140,7 @@ func (d *funcDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe. } return cursor, errors.ErrInvalidBeginningOfValue(buf[cursor], cursor) } + +func (d *funcDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: func decoder does not support decode path") +} diff --git a/internal/decoder/int.go b/internal/decoder/int.go index 509b753d..1a7f0819 100644 --- a/internal/decoder/int.go +++ b/internal/decoder/int.go @@ -240,3 +240,7 @@ func (d *intDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.P d.op(p, i64) return cursor, nil } + +func (d *intDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: int decoder does not support decode path") +} diff --git a/internal/decoder/interface.go b/internal/decoder/interface.go index 4dbb4be4..45c69ab8 100644 --- a/internal/decoder/interface.go +++ b/internal/decoder/interface.go @@ -94,6 +94,7 @@ func (d *interfaceDecoder) numDecoder(s *Stream) Decoder { var ( emptyInterfaceType = runtime.Type2RType(reflect.TypeOf((*interface{})(nil)).Elem()) + EmptyInterfaceType = emptyInterfaceType interfaceMapType = runtime.Type2RType( reflect.TypeOf((*map[string]interface{})(nil)).Elem(), ) @@ -456,3 +457,72 @@ func (d *interfaceDecoder) decodeEmptyInterface(ctx *RuntimeContext, cursor, dep } return cursor, errors.ErrInvalidBeginningOfValue(buf[cursor], cursor) } + +func NewPathDecoder() Decoder { + ifaceDecoder := &interfaceDecoder{ + typ: emptyInterfaceType, + structName: "", + fieldName: "", + floatDecoder: newFloatDecoder("", "", func(p unsafe.Pointer, v float64) { + *(*interface{})(p) = v + }), + numberDecoder: newNumberDecoder("", "", func(p unsafe.Pointer, v json.Number) { + *(*interface{})(p) = v + }), + stringDecoder: newStringDecoder("", ""), + } + ifaceDecoder.sliceDecoder = newSliceDecoder( + ifaceDecoder, + emptyInterfaceType, + emptyInterfaceType.Size(), + "", "", + ) + ifaceDecoder.mapDecoder = newMapDecoder( + interfaceMapType, + stringType, + ifaceDecoder.stringDecoder, + interfaceMapType.Elem(), + ifaceDecoder, + "", "", + ) + return ifaceDecoder +} + +var ( + truebytes = []byte("true") + falsebytes = []byte("false") +) + +func (d *interfaceDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + buf := ctx.Buf + cursor = skipWhiteSpace(buf, cursor) + switch buf[cursor] { + case '{': + return d.mapDecoder.DecodePath(ctx, cursor, depth) + case '[': + return d.sliceDecoder.DecodePath(ctx, cursor, depth) + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return d.floatDecoder.DecodePath(ctx, cursor, depth) + case '"': + return d.stringDecoder.DecodePath(ctx, cursor, depth) + case 't': + if err := validateTrue(buf, cursor); err != nil { + return nil, 0, err + } + cursor += 4 + return [][]byte{truebytes}, cursor, nil + case 'f': + if err := validateFalse(buf, cursor); err != nil { + return nil, 0, err + } + cursor += 5 + return [][]byte{falsebytes}, cursor, nil + case 'n': + if err := validateNull(buf, cursor); err != nil { + return nil, 0, err + } + cursor += 4 + return [][]byte{nullbytes}, cursor, nil + } + return nil, cursor, errors.ErrInvalidBeginningOfValue(buf[cursor], cursor) +} diff --git a/internal/decoder/invalid.go b/internal/decoder/invalid.go index 1ef50a7d..4c9721b0 100644 --- a/internal/decoder/invalid.go +++ b/internal/decoder/invalid.go @@ -43,3 +43,13 @@ func (d *invalidDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsa Field: d.fieldName, } } + +func (d *invalidDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, &errors.UnmarshalTypeError{ + Value: "object", + Type: runtime.RType2Type(d.typ), + Offset: cursor, + Struct: d.structName, + Field: d.fieldName, + } +} diff --git a/internal/decoder/map.go b/internal/decoder/map.go index cb55ef00..4ee8e5cc 100644 --- a/internal/decoder/map.go +++ b/internal/decoder/map.go @@ -185,3 +185,95 @@ func (d *mapDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.P cursor++ } } + +func (d *mapDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + buf := ctx.Buf + depth++ + if depth > maxDecodeNestingDepth { + return nil, 0, errors.ErrExceededMaxDepth(buf[cursor], cursor) + } + + cursor = skipWhiteSpace(buf, cursor) + buflen := int64(len(buf)) + if buflen < 2 { + return nil, 0, errors.ErrExpected("{} for map", cursor) + } + switch buf[cursor] { + case 'n': + if err := validateNull(buf, cursor); err != nil { + return nil, 0, err + } + cursor += 4 + return [][]byte{nullbytes}, cursor, nil + case '{': + default: + return nil, 0, errors.ErrExpected("{ character for map value", cursor) + } + cursor++ + cursor = skipWhiteSpace(buf, cursor) + if buf[cursor] == '}' { + cursor++ + return nil, cursor, nil + } + keyDecoder, ok := d.keyDecoder.(*stringDecoder) + if !ok { + return nil, 0, &errors.UnmarshalTypeError{ + Value: "string", + Type: reflect.TypeOf(""), + Offset: cursor, + Struct: d.structName, + Field: d.fieldName, + } + } + ret := [][]byte{} + for { + key, keyCursor, err := keyDecoder.decodeByte(buf, cursor) + if err != nil { + return nil, 0, err + } + cursor = skipWhiteSpace(buf, keyCursor) + if buf[cursor] != ':' { + return nil, 0, errors.ErrExpected("colon after object key", cursor) + } + cursor++ + child, found, err := ctx.Option.Path.Field(string(key)) + if err != nil { + return nil, 0, err + } + if found { + if child != nil { + oldPath := ctx.Option.Path.node + ctx.Option.Path.node = child + paths, c, err := d.valueDecoder.DecodePath(ctx, cursor, depth) + if err != nil { + return nil, 0, err + } + ctx.Option.Path.node = oldPath + ret = append(ret, paths...) + cursor = c + } else { + start := cursor + end, err := skipValue(buf, cursor, depth) + if err != nil { + return nil, 0, err + } + ret = append(ret, buf[start:end]) + cursor = end + } + } else { + c, err := skipValue(buf, cursor, depth) + if err != nil { + return nil, 0, err + } + cursor = skipWhiteSpace(buf, c) + } + if buf[cursor] == '}' { + cursor++ + return ret, cursor, nil + } + if buf[cursor] != ',' { + return nil, 0, errors.ErrExpected("comma after object value", cursor) + } + cursor++ + } +} diff --git a/internal/decoder/number.go b/internal/decoder/number.go index bf63773e..10e5435e 100644 --- a/internal/decoder/number.go +++ b/internal/decoder/number.go @@ -51,6 +51,17 @@ func (d *numberDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsaf return cursor, nil } +func (d *numberDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + bytes, c, err := d.decodeByte(ctx.Buf, cursor) + if err != nil { + return nil, 0, err + } + if bytes == nil { + return [][]byte{nullbytes}, c, nil + } + return [][]byte{bytes}, c, nil +} + func (d *numberDecoder) decodeStreamByte(s *Stream) ([]byte, error) { start := s.cursor for { diff --git a/internal/decoder/option.go b/internal/decoder/option.go index e41f876b..502f772e 100644 --- a/internal/decoder/option.go +++ b/internal/decoder/option.go @@ -7,9 +7,11 @@ type OptionFlags uint8 const ( FirstWinOption OptionFlags = 1 << iota ContextOption + PathOption ) type Option struct { Flags OptionFlags Context context.Context + Path *Path } diff --git a/internal/decoder/path.go b/internal/decoder/path.go new file mode 100644 index 00000000..33ddcd5b --- /dev/null +++ b/internal/decoder/path.go @@ -0,0 +1,653 @@ +package decoder + +import ( + "fmt" + "reflect" + "strconv" + + "github.com/goccy/go-json/internal/errors" + "github.com/goccy/go-json/internal/runtime" +) + +type PathString string + +func (s PathString) Build() (*Path, error) { + builder := new(PathBuilder) + return builder.Build([]rune(s)) +} + +type PathBuilder struct { + root PathNode + node PathNode + singleQuotePathSelector bool + doubleQuotePathSelector bool +} + +func (b *PathBuilder) Build(buf []rune) (*Path, error) { + node, err := b.build(buf) + if err != nil { + return nil, err + } + return &Path{ + node: node, + SingleQuotePathSelector: b.singleQuotePathSelector, + DoubleQuotePathSelector: b.doubleQuotePathSelector, + }, nil +} + +func (b *PathBuilder) build(buf []rune) (PathNode, error) { + if len(buf) == 0 { + return nil, errors.ErrEmptyPath() + } + if buf[0] != '$' { + return nil, errors.ErrInvalidPath("JSON Path must start with a $ character") + } + if len(buf) == 1 { + return nil, nil + } + buf = buf[1:] + offset, err := b.buildNext(buf) + if err != nil { + return nil, err + } + if len(buf) > offset { + return nil, errors.ErrInvalidPath("remain invalid path %q", buf[offset:]) + } + return b.root, nil +} + +func (b *PathBuilder) buildNextCharIfExists(buf []rune, cursor int) (int, error) { + if len(buf) > cursor+1 { + offset, err := b.buildNext(buf[cursor+1:]) + if err != nil { + return 0, err + } + return cursor + 1 + offset, nil + } + return cursor, nil +} + +func (b *PathBuilder) buildNext(buf []rune) (int, error) { + switch buf[0] { + case '.': + if len(buf) == 1 { + return 0, errors.ErrInvalidPath("JSON Path ends with dot character") + } + offset, err := b.buildSelector(buf[1:]) + if err != nil { + return 0, err + } + return offset + 1, nil + case '[': + if len(buf) == 1 { + return 0, errors.ErrInvalidPath("JSON Path ends with left bracket character") + } + offset, err := b.buildIndex(buf[1:]) + if err != nil { + return 0, err + } + return offset + 1, nil + default: + return 0, errors.ErrInvalidPath("expect dot or left bracket character. but found %c character", buf[0]) + } +} + +func (b *PathBuilder) buildSelector(buf []rune) (int, error) { + switch buf[0] { + case '.': + if len(buf) == 1 { + return 0, errors.ErrInvalidPath("JSON Path ends with double dot character") + } + offset, err := b.buildPathRecursive(buf[1:]) + if err != nil { + return 0, err + } + return 1 + offset, nil + case '[', ']', '$', '*': + return 0, errors.ErrInvalidPath("found invalid path character %c after dot", buf[0]) + } + for cursor := 0; cursor < len(buf); cursor++ { + switch buf[cursor] { + case '$', '*', ']': + return 0, errors.ErrInvalidPath("found %c character in field selector context", buf[cursor]) + case '.': + if cursor+1 >= len(buf) { + return 0, errors.ErrInvalidPath("JSON Path ends with dot character") + } + selector := buf[:cursor] + b.addSelectorNode(string(selector)) + offset, err := b.buildSelector(buf[cursor+1:]) + if err != nil { + return 0, err + } + return cursor + 1 + offset, nil + case '[': + if cursor+1 >= len(buf) { + return 0, errors.ErrInvalidPath("JSON Path ends with left bracket character") + } + selector := buf[:cursor] + b.addSelectorNode(string(selector)) + offset, err := b.buildIndex(buf[cursor+1:]) + if err != nil { + return 0, err + } + return cursor + 1 + offset, nil + case '"': + if cursor+1 >= len(buf) { + return 0, errors.ErrInvalidPath("JSON Path ends with double quote character") + } + offset, err := b.buildQuoteSelector(buf[cursor+1:], DoubleQuotePathSelector) + if err != nil { + return 0, err + } + return cursor + 1 + offset, nil + } + } + b.addSelectorNode(string(buf)) + return len(buf), nil +} + +func (b *PathBuilder) buildQuoteSelector(buf []rune, sel QuotePathSelector) (int, error) { + switch buf[0] { + case '[', ']', '$', '.', '*', '\'', '"': + return 0, errors.ErrInvalidPath("found invalid path character %c after quote", buf[0]) + } + for cursor := 0; cursor < len(buf); cursor++ { + switch buf[cursor] { + case '\'': + if sel != SingleQuotePathSelector { + return 0, errors.ErrInvalidPath("found double quote character in field selector with single quote context") + } + if len(buf) <= cursor+1 { + return 0, errors.ErrInvalidPath("JSON Path ends with single quote character in field selector context") + } + if buf[cursor+1] != ']' { + return 0, errors.ErrInvalidPath("expect right bracket for field selector with single quote but found %c", buf[cursor+1]) + } + selector := buf[:cursor] + b.addSelectorNode(string(selector)) + return b.buildNextCharIfExists(buf, cursor+1) + case '"': + if sel != DoubleQuotePathSelector { + return 0, errors.ErrInvalidPath("found single quote character in field selector with double quote context") + } + selector := buf[:cursor] + b.addSelectorNode(string(selector)) + return b.buildNextCharIfExists(buf, cursor) + } + } + return 0, errors.ErrInvalidPath("couldn't find quote character in selector quote path context") +} + +func (b *PathBuilder) buildPathRecursive(buf []rune) (int, error) { + switch buf[0] { + case '.', '[', ']', '$', '*': + return 0, errors.ErrInvalidPath("found invalid path character %c after double dot", buf[0]) + } + for cursor := 0; cursor < len(buf); cursor++ { + switch buf[cursor] { + case '$', '*', ']': + return 0, errors.ErrInvalidPath("found %c character in field selector context", buf[cursor]) + case '.': + if cursor+1 >= len(buf) { + return 0, errors.ErrInvalidPath("JSON Path ends with dot character") + } + selector := buf[:cursor] + b.addRecursiveNode(string(selector)) + offset, err := b.buildSelector(buf[cursor+1:]) + if err != nil { + return 0, err + } + return cursor + 1 + offset, nil + case '[': + if cursor+1 >= len(buf) { + return 0, errors.ErrInvalidPath("JSON Path ends with left bracket character") + } + selector := buf[:cursor] + b.addRecursiveNode(string(selector)) + offset, err := b.buildIndex(buf[cursor+1:]) + if err != nil { + return 0, err + } + return cursor + 1 + offset, nil + } + } + b.addRecursiveNode(string(buf)) + return len(buf), nil +} + +func (b *PathBuilder) buildIndex(buf []rune) (int, error) { + switch buf[0] { + case '.', '[', ']', '$': + return 0, errors.ErrInvalidPath("found invalid path character %c after left bracket", buf[0]) + case '\'': + if len(buf) == 1 { + return 0, errors.ErrInvalidPath("JSON Path ends with single quote character") + } + offset, err := b.buildQuoteSelector(buf[1:], SingleQuotePathSelector) + if err != nil { + return 0, err + } + return 1 + offset, nil + case '*': + if len(buf) == 1 { + return 0, errors.ErrInvalidPath("JSON Path ends with star character") + } + if buf[1] != ']' { + return 0, errors.ErrInvalidPath("expect right bracket character for index all path but found %c character", buf[1]) + } + b.addIndexAllNode() + offset := len("*]") + if len(buf) > 2 { + buildOffset, err := b.buildNext(buf[2:]) + if err != nil { + return 0, err + } + return offset + buildOffset, nil + } + return offset, nil + } + + for cursor := 0; cursor < len(buf); cursor++ { + switch buf[cursor] { + case ']': + index, err := strconv.ParseInt(string(buf[:cursor]), 10, 64) + if err != nil { + return 0, errors.ErrInvalidPath("%q is unexpected index path", buf[:cursor]) + } + b.addIndexNode(int(index)) + return b.buildNextCharIfExists(buf, cursor) + } + } + return 0, errors.ErrInvalidPath("couldn't find right bracket character in index path context") +} + +func (b *PathBuilder) addIndexAllNode() { + node := newPathIndexAllNode() + if b.root == nil { + b.root = node + b.node = node + } else { + b.node = b.node.chain(node) + } +} + +func (b *PathBuilder) addRecursiveNode(selector string) { + node := newPathRecursiveNode(selector) + if b.root == nil { + b.root = node + b.node = node + } else { + b.node = b.node.chain(node) + } +} + +func (b *PathBuilder) addSelectorNode(name string) { + node := newPathSelectorNode(name) + if b.root == nil { + b.root = node + b.node = node + } else { + b.node = b.node.chain(node) + } +} + +func (b *PathBuilder) addIndexNode(idx int) { + node := newPathIndexNode(idx) + if b.root == nil { + b.root = node + b.node = node + } else { + b.node = b.node.chain(node) + } +} + +type QuotePathSelector int + +const ( + SingleQuotePathSelector QuotePathSelector = 1 + DoubleQuotePathSelector QuotePathSelector = 2 +) + +type Path struct { + node PathNode + SingleQuotePathSelector bool + DoubleQuotePathSelector bool +} + +func (p *Path) Field(sel string) (PathNode, bool, error) { + return p.node.Field(sel) +} + +func (p *Path) Get(src, dst reflect.Value) error { + return p.node.Get(src, dst) +} + +type PathNode interface { + fmt.Stringer + Index(idx int) (PathNode, bool, error) + Field(fieldName string) (PathNode, bool, error) + Get(src, dst reflect.Value) error + chain(PathNode) PathNode + target() bool + single() bool +} + +type BasePathNode struct { + child PathNode +} + +func (n *BasePathNode) chain(node PathNode) PathNode { + n.child = node + return node +} + +func (n *BasePathNode) target() bool { + return n.child == nil +} + +func (n *BasePathNode) single() bool { + return true +} + +type PathSelectorNode struct { + *BasePathNode + selector string +} + +func newPathSelectorNode(selector string) *PathSelectorNode { + return &PathSelectorNode{ + BasePathNode: &BasePathNode{}, + selector: selector, + } +} + +func (n *PathSelectorNode) Index(idx int) (PathNode, bool, error) { + return nil, false, &errors.PathError{} +} + +func (n *PathSelectorNode) Field(fieldName string) (PathNode, bool, error) { + if n.selector == fieldName { + return n.child, true, nil + } + return nil, false, nil +} + +func (n *PathSelectorNode) Get(src, dst reflect.Value) error { + switch src.Type().Kind() { + case reflect.Map: + iter := src.MapRange() + for iter.Next() { + key, ok := iter.Key().Interface().(string) + if !ok { + return fmt.Errorf("invalid map key type %T", src.Type().Key()) + } + child, found, err := n.Field(key) + if err != nil { + return err + } + if found { + if child != nil { + return child.Get(iter.Value(), dst) + } + return AssignValue(iter.Value(), dst) + } + } + case reflect.Struct: + typ := src.Type() + for i := 0; i < typ.Len(); i++ { + tag := runtime.StructTagFromField(typ.Field(i)) + child, found, err := n.Field(tag.Key) + if err != nil { + return err + } + if found { + if child != nil { + return child.Get(src.Field(i), dst) + } + return AssignValue(src.Field(i), dst) + } + } + case reflect.Ptr: + return n.Get(src.Elem(), dst) + case reflect.Interface: + return n.Get(reflect.ValueOf(src.Interface()), dst) + case reflect.Float64, reflect.String, reflect.Bool: + return AssignValue(src, dst) + } + return fmt.Errorf("failed to get %s value from %s", n.selector, src.Type()) +} + +func (n *PathSelectorNode) String() string { + s := fmt.Sprintf(".%s", n.selector) + if n.child != nil { + s += n.child.String() + } + return s +} + +type PathIndexNode struct { + *BasePathNode + selector int +} + +func newPathIndexNode(selector int) *PathIndexNode { + return &PathIndexNode{ + BasePathNode: &BasePathNode{}, + selector: selector, + } +} + +func (n *PathIndexNode) Index(idx int) (PathNode, bool, error) { + if n.selector == idx { + return n.child, true, nil + } + return nil, false, nil +} + +func (n *PathIndexNode) Field(fieldName string) (PathNode, bool, error) { + return nil, false, &errors.PathError{} +} + +func (n *PathIndexNode) Get(src, dst reflect.Value) error { + switch src.Type().Kind() { + case reflect.Array, reflect.Slice: + if src.Len() > n.selector { + if n.child != nil { + return n.child.Get(src.Index(n.selector), dst) + } + return AssignValue(src.Index(n.selector), dst) + } + case reflect.Ptr: + return n.Get(src.Elem(), dst) + case reflect.Interface: + return n.Get(reflect.ValueOf(src.Interface()), dst) + } + return fmt.Errorf("failed to get [%d] value from %s", n.selector, src.Type()) +} + +func (n *PathIndexNode) String() string { + s := fmt.Sprintf("[%d]", n.selector) + if n.child != nil { + s += n.child.String() + } + return s +} + +type PathIndexAllNode struct { + *BasePathNode +} + +func newPathIndexAllNode() *PathIndexAllNode { + return &PathIndexAllNode{ + BasePathNode: &BasePathNode{}, + } +} + +func (n *PathIndexAllNode) Index(idx int) (PathNode, bool, error) { + return n.child, true, nil +} + +func (n *PathIndexAllNode) Field(fieldName string) (PathNode, bool, error) { + return nil, false, &errors.PathError{} +} + +func (n *PathIndexAllNode) Get(src, dst reflect.Value) error { + switch src.Type().Kind() { + case reflect.Array, reflect.Slice: + var arr []interface{} + for i := 0; i < src.Len(); i++ { + var v interface{} + rv := reflect.ValueOf(&v) + if n.child != nil { + if err := n.child.Get(src.Index(i), rv); err != nil { + return err + } + } else { + if err := AssignValue(src.Index(i), rv); err != nil { + return err + } + } + arr = append(arr, v) + } + if err := AssignValue(reflect.ValueOf(arr), dst); err != nil { + return err + } + return nil + case reflect.Ptr: + return n.Get(src.Elem(), dst) + case reflect.Interface: + return n.Get(reflect.ValueOf(src.Interface()), dst) + } + return fmt.Errorf("failed to get all value from %s", src.Type()) +} + +func (n *PathIndexAllNode) String() string { + s := "[*]" + if n.child != nil { + s += n.child.String() + } + return s +} + +type PathRecursiveNode struct { + *BasePathNode + selector string +} + +func newPathRecursiveNode(selector string) *PathRecursiveNode { + node := newPathSelectorNode(selector) + return &PathRecursiveNode{ + BasePathNode: &BasePathNode{ + child: node, + }, + selector: selector, + } +} + +func (n *PathRecursiveNode) Field(fieldName string) (PathNode, bool, error) { + if n.selector == fieldName { + return n.child, true, nil + } + return nil, false, nil +} + +func (n *PathRecursiveNode) Index(_ int) (PathNode, bool, error) { + return n, true, nil +} + +func valueToSliceValue(v interface{}) []interface{} { + rv := reflect.ValueOf(v) + ret := []interface{}{} + if rv.Type().Kind() == reflect.Slice || rv.Type().Kind() == reflect.Array { + for i := 0; i < rv.Len(); i++ { + ret = append(ret, rv.Index(i).Interface()) + } + return ret + } + return []interface{}{v} +} + +func (n *PathRecursiveNode) Get(src, dst reflect.Value) error { + if n.child == nil { + return fmt.Errorf("failed to get by recursive path ..%s", n.selector) + } + var arr []interface{} + switch src.Type().Kind() { + case reflect.Map: + iter := src.MapRange() + for iter.Next() { + key, ok := iter.Key().Interface().(string) + if !ok { + return fmt.Errorf("invalid map key type %T", src.Type().Key()) + } + child, found, err := n.Field(key) + if err != nil { + return err + } + if found { + var v interface{} + rv := reflect.ValueOf(&v) + _ = child.Get(iter.Value(), rv) + arr = append(arr, valueToSliceValue(v)...) + } else { + var v interface{} + rv := reflect.ValueOf(&v) + _ = n.Get(iter.Value(), rv) + if v != nil { + arr = append(arr, valueToSliceValue(v)...) + } + } + } + _ = AssignValue(reflect.ValueOf(arr), dst) + return nil + case reflect.Struct: + typ := src.Type() + for i := 0; i < typ.Len(); i++ { + tag := runtime.StructTagFromField(typ.Field(i)) + child, found, err := n.Field(tag.Key) + if err != nil { + return err + } + if found { + var v interface{} + rv := reflect.ValueOf(&v) + _ = child.Get(src.Field(i), rv) + arr = append(arr, valueToSliceValue(v)...) + } else { + var v interface{} + rv := reflect.ValueOf(&v) + _ = n.Get(src.Field(i), rv) + if v != nil { + arr = append(arr, valueToSliceValue(v)...) + } + } + } + _ = AssignValue(reflect.ValueOf(arr), dst) + return nil + case reflect.Array, reflect.Slice: + for i := 0; i < src.Len(); i++ { + var v interface{} + rv := reflect.ValueOf(&v) + _ = n.Get(src.Index(i), rv) + if v != nil { + arr = append(arr, valueToSliceValue(v)...) + } + } + _ = AssignValue(reflect.ValueOf(arr), dst) + return nil + case reflect.Ptr: + return n.Get(src.Elem(), dst) + case reflect.Interface: + return n.Get(reflect.ValueOf(src.Interface()), dst) + } + return fmt.Errorf("failed to get %s value from %s", n.selector, src.Type()) +} + +func (n *PathRecursiveNode) String() string { + s := fmt.Sprintf("..%s", n.selector) + if n.child != nil { + s += n.child.String() + } + return s +} diff --git a/internal/decoder/ptr.go b/internal/decoder/ptr.go index 2c83b9c4..de12e105 100644 --- a/internal/decoder/ptr.go +++ b/internal/decoder/ptr.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "unsafe" "github.com/goccy/go-json/internal/runtime" @@ -34,6 +35,10 @@ func (d *ptrDecoder) contentDecoder() Decoder { //go:linkname unsafe_New reflect.unsafe_New func unsafe_New(*runtime.Type) unsafe.Pointer +func UnsafeNew(t *runtime.Type) unsafe.Pointer { + return unsafe_New(t) +} + func (d *ptrDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) error { if s.skipWhiteSpace() == nul { s.read() @@ -85,3 +90,7 @@ func (d *ptrDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.P cursor = c return cursor, nil } + +func (d *ptrDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: ptr decoder does not support decode path") +} diff --git a/internal/decoder/slice.go b/internal/decoder/slice.go index 85b6e111..ec4dcd33 100644 --- a/internal/decoder/slice.go +++ b/internal/decoder/slice.go @@ -299,3 +299,80 @@ func (d *sliceDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe } } } + +func (d *sliceDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + buf := ctx.Buf + depth++ + if depth > maxDecodeNestingDepth { + return nil, 0, errors.ErrExceededMaxDepth(buf[cursor], cursor) + } + + ret := [][]byte{} + for { + switch buf[cursor] { + case ' ', '\n', '\t', '\r': + cursor++ + continue + case 'n': + if err := validateNull(buf, cursor); err != nil { + return nil, 0, err + } + cursor += 4 + return [][]byte{nullbytes}, cursor, nil + case '[': + cursor++ + cursor = skipWhiteSpace(buf, cursor) + if buf[cursor] == ']' { + cursor++ + return ret, cursor, nil + } + idx := 0 + for { + child, found, err := ctx.Option.Path.node.Index(idx) + if err != nil { + return nil, 0, err + } + if found { + if child != nil { + oldPath := ctx.Option.Path.node + ctx.Option.Path.node = child + paths, c, err := d.valueDecoder.DecodePath(ctx, cursor, depth) + if err != nil { + return nil, 0, err + } + ctx.Option.Path.node = oldPath + ret = append(ret, paths...) + cursor = c + } else { + start := cursor + end, err := skipValue(buf, cursor, depth) + if err != nil { + return nil, 0, err + } + ret = append(ret, buf[start:end]) + } + } else { + c, err := skipValue(buf, cursor, depth) + if err != nil { + return nil, 0, err + } + cursor = skipWhiteSpace(buf, c) + } + switch buf[cursor] { + case ']': + cursor++ + return ret, cursor, nil + case ',': + idx++ + default: + return nil, 0, errors.ErrInvalidCharacter(buf[cursor], "slice", cursor) + } + cursor++ + } + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + return nil, 0, d.errNumber(cursor) + default: + return nil, 0, errors.ErrUnexpectedEndOfJSON("slice", cursor) + } + } +} diff --git a/internal/decoder/string.go b/internal/decoder/string.go index d07ad710..32602c90 100644 --- a/internal/decoder/string.go +++ b/internal/decoder/string.go @@ -60,6 +60,17 @@ func (d *stringDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsaf return cursor, nil } +func (d *stringDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + bytes, c, err := d.decodeByte(ctx.Buf, cursor) + if err != nil { + return nil, 0, err + } + if bytes == nil { + return [][]byte{nullbytes}, c, nil + } + return [][]byte{bytes}, c, nil +} + var ( hexToInt = [256]int{ '0': 0, diff --git a/internal/decoder/struct.go b/internal/decoder/struct.go index 2c646804..6d326548 100644 --- a/internal/decoder/struct.go +++ b/internal/decoder/struct.go @@ -817,3 +817,7 @@ func (d *structDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsaf cursor++ } } + +func (d *structDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: struct decoder does not support decode path") +} diff --git a/internal/decoder/type.go b/internal/decoder/type.go index 70e9907c..beaf3ab8 100644 --- a/internal/decoder/type.go +++ b/internal/decoder/type.go @@ -10,6 +10,7 @@ import ( type Decoder interface { Decode(*RuntimeContext, int64, int64, unsafe.Pointer) (int64, error) + DecodePath(*RuntimeContext, int64, int64) ([][]byte, int64, error) DecodeStream(*Stream, int64, unsafe.Pointer) error } diff --git a/internal/decoder/uint.go b/internal/decoder/uint.go index a62c5149..4131731b 100644 --- a/internal/decoder/uint.go +++ b/internal/decoder/uint.go @@ -188,3 +188,7 @@ func (d *uintDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe. d.op(p, u64) return cursor, nil } + +func (d *uintDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: uint decoder does not support decode path") +} diff --git a/internal/decoder/unmarshal_json.go b/internal/decoder/unmarshal_json.go index e9b25c68..4cd6dbd5 100644 --- a/internal/decoder/unmarshal_json.go +++ b/internal/decoder/unmarshal_json.go @@ -3,6 +3,7 @@ package decoder import ( "context" "encoding/json" + "fmt" "unsafe" "github.com/goccy/go-json/internal/errors" @@ -97,3 +98,7 @@ func (d *unmarshalJSONDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, } return end, nil } + +func (d *unmarshalJSONDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: unmarshal json decoder does not support decode path") +} diff --git a/internal/decoder/unmarshal_text.go b/internal/decoder/unmarshal_text.go index 1ef28778..6d37993f 100644 --- a/internal/decoder/unmarshal_text.go +++ b/internal/decoder/unmarshal_text.go @@ -3,6 +3,7 @@ package decoder import ( "bytes" "encoding" + "fmt" "unicode" "unicode/utf16" "unicode/utf8" @@ -142,6 +143,10 @@ func (d *unmarshalTextDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, return end, nil } +func (d *unmarshalTextDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: unmarshal text decoder does not support decode path") +} + func unquoteBytes(s []byte) (t []byte, ok bool) { length := len(s) if length < 2 || s[0] != '"' || s[length-1] != '"' { diff --git a/internal/decoder/wrapped_string.go b/internal/decoder/wrapped_string.go index 66227ae0..0c4e2e6e 100644 --- a/internal/decoder/wrapped_string.go +++ b/internal/decoder/wrapped_string.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "reflect" "unsafe" @@ -66,3 +67,7 @@ func (d *wrappedStringDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, ctx.Buf = oldBuf return c, nil } + +func (d *wrappedStringDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]byte, int64, error) { + return nil, 0, fmt.Errorf("json: wrapped string decoder does not support decode path") +} diff --git a/internal/errors/error.go b/internal/errors/error.go index d58e39f4..9207d0ff 100644 --- a/internal/errors/error.go +++ b/internal/errors/error.go @@ -162,3 +162,22 @@ func ErrInvalidBeginningOfValue(c byte, cursor int64) *SyntaxError { Offset: cursor, } } + +type PathError struct { + msg string +} + +func (e *PathError) Error() string { + return fmt.Sprintf("json: invalid path format: %s", e.msg) +} + +func ErrInvalidPath(msg string, args ...interface{}) *PathError { + if len(args) != 0 { + return &PathError{msg: fmt.Sprintf(msg, args...)} + } + return &PathError{msg: msg} +} + +func ErrEmptyPath() *PathError { + return &PathError{msg: "path is empty"} +} diff --git a/path.go b/path.go new file mode 100644 index 00000000..0b9c68cf --- /dev/null +++ b/path.go @@ -0,0 +1,73 @@ +package json + +import ( + "reflect" + + "github.com/goccy/go-json/internal/decoder" +) + +// CreatePath creates JSON Path. +// +// JSON Path rule +// $ : root object or element. The JSON Path format must start with this operator, which refers to the outermost level of the JSON-formatted string. +// . : child operator. You can identify child values using dot-notation. +// .. : recursive descent. +// [] : subscript operator. If the JSON object is an array, you can use brackets to specify the array index. +// [*] : all objects/elements for array. +// +// Reserved words must be properly escaped when included in Path. +// Escale Rule +// single quote style escape: e.g.) `$['a.b'].c` +// double quote style escape: e.g.) `$."a.b".c` +func CreatePath(p string) (*Path, error) { + path, err := decoder.PathString(p).Build() + if err != nil { + return nil, err + } + return &Path{path: path}, nil +} + +// Path represents JSON Path. +type Path struct { + path *decoder.Path +} + +// UsedSingleQuotePathSelector whether single quote-based escaping was done when building the JSON Path. +func (p *Path) UsedSingleQuotePathSelector() bool { + return p.path.SingleQuotePathSelector +} + +// UsedSingleQuotePathSelector whether double quote-based escaping was done when building the JSON Path. +func (p *Path) UsedDoubleQuotePathSelector() bool { + return p.path.DoubleQuotePathSelector +} + +// Extract extracts a specific JSON string. +func (p *Path) Extract(data []byte, optFuncs ...DecodeOptionFunc) ([][]byte, error) { + return extractFromPath(p, data, optFuncs...) +} + +// Unmarshal extract and decode the value of the part corresponding to JSON Path from the input data. +func (p *Path) Unmarshal(data []byte, v interface{}, optFuncs ...DecodeOptionFunc) error { + contents, err := extractFromPath(p, data, optFuncs...) + if err != nil { + return err + } + results := make([]interface{}, 0, len(contents)) + for _, content := range contents { + var result interface{} + if err := Unmarshal(content, &result); err != nil { + return err + } + results = append(results, result) + } + if err := decoder.AssignValue(reflect.ValueOf(results), reflect.ValueOf(v)); err != nil { + return err + } + return nil +} + +// Get extract and substitute the value of the part corresponding to JSON Path from the input value. +func (p *Path) Get(src, dst interface{}) error { + return p.path.Get(reflect.ValueOf(src), reflect.ValueOf(dst)) +} diff --git a/path_test.go b/path_test.go new file mode 100644 index 00000000..4891eb8f --- /dev/null +++ b/path_test.go @@ -0,0 +1,234 @@ +package json_test + +import ( + "bytes" + "reflect" + "sort" + "testing" + + "github.com/goccy/go-json" +) + +func TestExtractPath(t *testing.T) { + src := []byte(`{"a":{"b":10,"c":true},"b":"text"}`) + t.Run("$.a.b", func(t *testing.T) { + path, err := json.CreatePath("$.a.b") + if err != nil { + t.Fatal(err) + } + contents, err := path.Extract(src) + if err != nil { + t.Fatal(err) + } + if len(contents) != 1 { + t.Fatal("failed to extract") + } + if !bytes.Equal(contents[0], []byte("10")) { + t.Fatal("failed to extract") + } + }) + t.Run("$.b", func(t *testing.T) { + path, err := json.CreatePath("$.b") + if err != nil { + t.Fatal(err) + } + contents, err := path.Extract(src) + if err != nil { + t.Fatal(err) + } + if len(contents) != 1 { + t.Fatal("failed to extract") + } + if !bytes.Equal(contents[0], []byte(`"text"`)) { + t.Fatal("failed to extract") + } + }) + t.Run("$.a", func(t *testing.T) { + path, err := json.CreatePath("$.a") + if err != nil { + t.Fatal(err) + } + contents, err := path.Extract(src) + if err != nil { + t.Fatal(err) + } + if len(contents) != 1 { + t.Fatal("failed to extract") + } + if !bytes.Equal(contents[0], []byte(`{"b":10,"c":true}`)) { + t.Fatal("failed to extract") + } + }) +} + +func TestUnmarshalPath(t *testing.T) { + t.Run("int", func(t *testing.T) { + src := []byte(`{"a":{"b":10,"c":true},"b":"text"}`) + t.Run("success", func(t *testing.T) { + path, err := json.CreatePath("$.a.b") + if err != nil { + t.Fatal(err) + } + var v int + if err := path.Unmarshal(src, &v); err != nil { + t.Fatal(err) + } + if v != 10 { + t.Fatal("failed to unmarshal path") + } + }) + t.Run("failure", func(t *testing.T) { + path, err := json.CreatePath("$.a.c") + if err != nil { + t.Fatal(err) + } + var v map[string]interface{} + if err := path.Unmarshal(src, &v); err == nil { + t.Fatal("expected error") + } + }) + }) + t.Run("bool", func(t *testing.T) { + src := []byte(`{"a":{"b":10,"c":true},"b":"text"}`) + t.Run("success", func(t *testing.T) { + path, err := json.CreatePath("$.a.c") + if err != nil { + t.Fatal(err) + } + var v bool + if err := path.Unmarshal(src, &v); err != nil { + t.Fatal(err) + } + if !v { + t.Fatal("failed to unmarshal path") + } + }) + t.Run("failure", func(t *testing.T) { + path, err := json.CreatePath("$.a.b") + if err != nil { + t.Fatal(err) + } + var v bool + if err := path.Unmarshal(src, &v); err == nil { + t.Fatal("expected error") + } + }) + }) + t.Run("map", func(t *testing.T) { + src := []byte(`{"a":{"b":10,"c":true},"b":"text"}`) + t.Run("success", func(t *testing.T) { + path, err := json.CreatePath("$.a") + if err != nil { + t.Fatal(err) + } + var v map[string]interface{} + if err := path.Unmarshal(src, &v); err != nil { + t.Fatal(err) + } + if len(v) != 2 { + t.Fatal("failed to decode map") + } + }) + }) + + t.Run("path with single quote selector", func(t *testing.T) { + path, err := json.CreatePath("$['a.b'].c") + if err != nil { + t.Fatal(err) + } + + var v string + if err := path.Unmarshal([]byte(`{"a.b": {"c": "world"}}`), &v); err != nil { + t.Fatal(err) + } + if v != "world" { + t.Fatal("failed to unmarshal path") + } + }) + t.Run("path with double quote selector", func(t *testing.T) { + path, err := json.CreatePath(`$."a.b".c`) + if err != nil { + t.Fatal(err) + } + + var v string + if err := path.Unmarshal([]byte(`{"a.b": {"c": "world"}}`), &v); err != nil { + t.Fatal(err) + } + if v != "world" { + t.Fatal("failed to unmarshal path") + } + }) +} + +func TestGetPath(t *testing.T) { + t.Run("selector", func(t *testing.T) { + var v interface{} + if err := json.Unmarshal([]byte(`{"a":{"b":10,"c":true},"b":"text"}`), &v); err != nil { + t.Fatal(err) + } + path, err := json.CreatePath("$.a.b") + if err != nil { + t.Fatal(err) + } + var b int + if err := path.Get(v, &b); err != nil { + t.Fatal(err) + } + if b != 10 { + t.Fatalf("failed to decode by json.Get") + } + }) + t.Run("index", func(t *testing.T) { + var v interface{} + if err := json.Unmarshal([]byte(`{"a":[{"b":10,"c":true},{"b":"text"}]}`), &v); err != nil { + t.Fatal(err) + } + path, err := json.CreatePath("$.a[0].b") + if err != nil { + t.Fatal(err) + } + var b int + if err := path.Get(v, &b); err != nil { + t.Fatal(err) + } + if b != 10 { + t.Fatalf("failed to decode by json.Get") + } + }) + t.Run("indexAll", func(t *testing.T) { + var v interface{} + if err := json.Unmarshal([]byte(`{"a":[{"b":1,"c":true},{"b":2},{"b":3}]}`), &v); err != nil { + t.Fatal(err) + } + path, err := json.CreatePath("$.a[*].b") + if err != nil { + t.Fatal(err) + } + var b []int + if err := path.Get(v, &b); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(b, []int{1, 2, 3}) { + t.Fatalf("failed to decode by json.Get") + } + }) + t.Run("recursive", func(t *testing.T) { + var v interface{} + if err := json.Unmarshal([]byte(`{"a":[{"b":1,"c":true},{"b":2},{"b":3}],"a2":{"b":4}}`), &v); err != nil { + t.Fatal(err) + } + path, err := json.CreatePath("$..b") + if err != nil { + t.Fatal(err) + } + var b []int + if err := path.Get(v, &b); err != nil { + t.Fatal(err) + } + sort.Ints(b) + if !reflect.DeepEqual(b, []int{1, 2, 3, 4}) { + t.Fatalf("failed to decode by json.Get") + } + }) +} From d8329d56f943b383f30877ebfb24603f3910673b Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 29 Nov 2022 04:07:50 +0900 Subject: [PATCH 42/43] Update path.go --- path.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/path.go b/path.go index 0b9c68cf..68bb5a23 100644 --- a/path.go +++ b/path.go @@ -16,7 +16,8 @@ import ( // [*] : all objects/elements for array. // // Reserved words must be properly escaped when included in Path. -// Escale Rule +// +// Escape Rule // single quote style escape: e.g.) `$['a.b'].c` // double quote style escape: e.g.) `$."a.b".c` func CreatePath(p string) (*Path, error) { From 1de494fd9a90d81c3ec365dc351d7e4e49d4a044 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 29 Nov 2022 21:44:55 +0900 Subject: [PATCH 43/43] Fix json path --- decode.go | 3 +++ internal/decoder/map.go | 3 ++- internal/decoder/path.go | 27 ++++++++++++++++++++++----- internal/decoder/slice.go | 4 +++- path.go | 10 ++++++++++ 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/decode.go b/decode.go index b1512cec..74c6ac3b 100644 --- a/decode.go +++ b/decode.go @@ -88,6 +88,9 @@ var ( ) func extractFromPath(path *Path, data []byte, optFuncs ...DecodeOptionFunc) ([][]byte, error) { + if path.path.RootSelectorOnly { + return [][]byte{data}, nil + } src := make([]byte, len(data)+1) // append nul byte to the end copy(src, data) diff --git a/internal/decoder/map.go b/internal/decoder/map.go index 4ee8e5cc..7a6eea34 100644 --- a/internal/decoder/map.go +++ b/internal/decoder/map.go @@ -265,8 +265,9 @@ func (d *mapDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][]b if err != nil { return nil, 0, err } - cursor = skipWhiteSpace(buf, c) + cursor = c } + cursor = skipWhiteSpace(buf, cursor) if buf[cursor] == '}' { cursor++ return ret, cursor, nil diff --git a/internal/decoder/path.go b/internal/decoder/path.go index 33ddcd5b..a15ff69e 100644 --- a/internal/decoder/path.go +++ b/internal/decoder/path.go @@ -30,6 +30,7 @@ func (b *PathBuilder) Build(buf []rune) (*Path, error) { } return &Path{ node: node, + RootSelectorOnly: node == nil, SingleQuotePathSelector: b.singleQuotePathSelector, DoubleQuotePathSelector: b.doubleQuotePathSelector, }, nil @@ -57,8 +58,8 @@ func (b *PathBuilder) build(buf []rune) (PathNode, error) { } func (b *PathBuilder) buildNextCharIfExists(buf []rune, cursor int) (int, error) { - if len(buf) > cursor+1 { - offset, err := b.buildNext(buf[cursor+1:]) + if len(buf) > cursor { + offset, err := b.buildNext(buf[cursor:]) if err != nil { return 0, err } @@ -166,14 +167,16 @@ func (b *PathBuilder) buildQuoteSelector(buf []rune, sel QuotePathSelector) (int } selector := buf[:cursor] b.addSelectorNode(string(selector)) - return b.buildNextCharIfExists(buf, cursor+1) + b.singleQuotePathSelector = true + return b.buildNextCharIfExists(buf, cursor+2) case '"': if sel != DoubleQuotePathSelector { return 0, errors.ErrInvalidPath("found single quote character in field selector with double quote context") } selector := buf[:cursor] b.addSelectorNode(string(selector)) - return b.buildNextCharIfExists(buf, cursor) + b.doubleQuotePathSelector = true + return b.buildNextCharIfExists(buf, cursor+1) } } return 0, errors.ErrInvalidPath("couldn't find quote character in selector quote path context") @@ -256,7 +259,7 @@ func (b *PathBuilder) buildIndex(buf []rune) (int, error) { return 0, errors.ErrInvalidPath("%q is unexpected index path", buf[:cursor]) } b.addIndexNode(int(index)) - return b.buildNextCharIfExists(buf, cursor) + return b.buildNextCharIfExists(buf, cursor+1) } } return 0, errors.ErrInvalidPath("couldn't find right bracket character in index path context") @@ -311,18 +314,32 @@ const ( type Path struct { node PathNode + RootSelectorOnly bool SingleQuotePathSelector bool DoubleQuotePathSelector bool } func (p *Path) Field(sel string) (PathNode, bool, error) { + if p.node == nil { + return nil, false, nil + } return p.node.Field(sel) } func (p *Path) Get(src, dst reflect.Value) error { + if p.node == nil { + return nil + } return p.node.Get(src, dst) } +func (p *Path) String() string { + if p.node == nil { + return "$" + } + return p.node.String() +} + type PathNode interface { fmt.Stringer Index(idx int) (PathNode, bool, error) diff --git a/internal/decoder/slice.go b/internal/decoder/slice.go index ec4dcd33..30a23e4b 100644 --- a/internal/decoder/slice.go +++ b/internal/decoder/slice.go @@ -350,14 +350,16 @@ func (d *sliceDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int64) ([][ return nil, 0, err } ret = append(ret, buf[start:end]) + cursor = end } } else { c, err := skipValue(buf, cursor, depth) if err != nil { return nil, 0, err } - cursor = skipWhiteSpace(buf, c) + cursor = c } + cursor = skipWhiteSpace(buf, cursor) switch buf[cursor] { case ']': cursor++ diff --git a/path.go b/path.go index 68bb5a23..38abce78 100644 --- a/path.go +++ b/path.go @@ -33,6 +33,11 @@ type Path struct { path *decoder.Path } +// RootSelectorOnly whether only the root selector ($) is used. +func (p *Path) RootSelectorOnly() bool { + return p.path.RootSelectorOnly +} + // UsedSingleQuotePathSelector whether single quote-based escaping was done when building the JSON Path. func (p *Path) UsedSingleQuotePathSelector() bool { return p.path.SingleQuotePathSelector @@ -48,6 +53,11 @@ func (p *Path) Extract(data []byte, optFuncs ...DecodeOptionFunc) ([][]byte, err return extractFromPath(p, data, optFuncs...) } +// PathString returns original JSON Path string. +func (p *Path) PathString() string { + return p.path.String() +} + // Unmarshal extract and decode the value of the part corresponding to JSON Path from the input data. func (p *Path) Unmarshal(data []byte, v interface{}, optFuncs ...DecodeOptionFunc) error { contents, err := extractFromPath(p, data, optFuncs...)