From 50a60f932b09b5775ebd048782e0136f5e0f7942 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Tue, 29 Nov 2022 22:37:41 +0900 Subject: [PATCH 1/9] Update CHANGELOG --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d63009fd..c9941520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# v0.10.0 - 2022/11/29 + +### New features + +* Support JSON Path ( #250 ) + +### Fix bugs + +* Fix marshaler for map's key ( #409 ) + # v0.9.11 - 2022/08/18 ### Fix bugs From 1480e0046f450d506fa63d04db2730de149750bd Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Fri, 2 Dec 2022 01:51:29 +0900 Subject: [PATCH 2/9] Fix checkptr error for array decoder --- decode_test.go | 10 ++++++++++ internal/decoder/array.go | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/decode_test.go b/decode_test.go index 088367e5..f139eae5 100644 --- a/decode_test.go +++ b/decode_test.go @@ -4005,3 +4005,13 @@ func TestIssue384(t *testing.T) { } } } + +func TestIssue408(t *testing.T) { + type T struct { + Arr [2]int32 `json:"arr"` + } + var v T + if err := json.Unmarshal([]byte(`{"arr": [1,2]}`), &v); err != nil { + t.Fatal(err) + } +} diff --git a/internal/decoder/array.go b/internal/decoder/array.go index 8ef91cfa..4b23ed43 100644 --- a/internal/decoder/array.go +++ b/internal/decoder/array.go @@ -19,7 +19,9 @@ type arrayDecoder struct { } func newArrayDecoder(dec Decoder, elemType *runtime.Type, alen int, structName, fieldName string) *arrayDecoder { - zeroValue := *(*unsafe.Pointer)(unsafe_New(elemType)) + // workaround to avoid checkptr errors. cannot use `*(*unsafe.Pointer)(unsafe_New(elemType))` directly. + zeroValuePtr := unsafe_New(elemType) + zeroValue := **(**unsafe.Pointer)(unsafe.Pointer(&zeroValuePtr)) return &arrayDecoder{ valueDecoder: dec, elemType: elemType, From cdbc29239bfa2aaad1030e9a6c8c9a1b9d6b492e Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Wed, 22 Feb 2023 20:54:26 +0900 Subject: [PATCH 3/9] fix: added buffer size check when decoding key fix #429 --- decode_test.go | 15 ++++++++++++++ internal/decoder/struct.go | 42 +++++++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/decode_test.go b/decode_test.go index f139eae5..a1a10e9c 100644 --- a/decode_test.go +++ b/decode_test.go @@ -4015,3 +4015,18 @@ func TestIssue408(t *testing.T) { t.Fatal(err) } } + +func TestIssue429(t *testing.T) { + var x struct { + N int32 + } + for _, b := range []string{ + `{"\u"`, + `{"\u0"`, + `{"\u00"`, + } { + if err := json.Unmarshal([]byte(b), &x); err == nil { + t.Errorf("unexpected success") + } + } +} diff --git a/internal/decoder/struct.go b/internal/decoder/struct.go index 6d326548..4e14a20f 100644 --- a/internal/decoder/struct.go +++ b/internal/decoder/struct.go @@ -158,49 +158,53 @@ func (d *structDecoder) tryOptimize() { } // decode from '\uXXXX' -func decodeKeyCharByUnicodeRune(buf []byte, cursor int64) ([]byte, int64) { +func decodeKeyCharByUnicodeRune(buf []byte, cursor int64) ([]byte, int64, error) { const defaultOffset = 4 const surrogateOffset = 6 + if cursor+defaultOffset >= int64(len(buf)) { + return nil, 0, errors.ErrUnexpectedEndOfJSON("escaped string", cursor) + } + r := unicodeToRune(buf[cursor : cursor+defaultOffset]) if utf16.IsSurrogate(r) { cursor += defaultOffset if cursor+surrogateOffset >= int64(len(buf)) || buf[cursor] != '\\' || buf[cursor+1] != 'u' { - return []byte(string(unicode.ReplacementChar)), cursor + defaultOffset - 1 + return []byte(string(unicode.ReplacementChar)), cursor + defaultOffset - 1, nil } cursor += 2 r2 := unicodeToRune(buf[cursor : cursor+defaultOffset]) if r := utf16.DecodeRune(r, r2); r != unicode.ReplacementChar { - return []byte(string(r)), cursor + defaultOffset - 1 + return []byte(string(r)), cursor + defaultOffset - 1, nil } } - return []byte(string(r)), cursor + defaultOffset - 1 + return []byte(string(r)), cursor + defaultOffset - 1, nil } -func decodeKeyCharByEscapedChar(buf []byte, cursor int64) ([]byte, int64) { +func decodeKeyCharByEscapedChar(buf []byte, cursor int64) ([]byte, int64, error) { c := buf[cursor] cursor++ switch c { case '"': - return []byte{'"'}, cursor + return []byte{'"'}, cursor, nil case '\\': - return []byte{'\\'}, cursor + return []byte{'\\'}, cursor, nil case '/': - return []byte{'/'}, cursor + return []byte{'/'}, cursor, nil case 'b': - return []byte{'\b'}, cursor + return []byte{'\b'}, cursor, nil case 'f': - return []byte{'\f'}, cursor + return []byte{'\f'}, cursor, nil case 'n': - return []byte{'\n'}, cursor + return []byte{'\n'}, cursor, nil case 'r': - return []byte{'\r'}, cursor + return []byte{'\r'}, cursor, nil case 't': - return []byte{'\t'}, cursor + return []byte{'\t'}, cursor, nil case 'u': return decodeKeyCharByUnicodeRune(buf, cursor) } - return nil, cursor + return nil, cursor, nil } func decodeKeyByBitmapUint8(d *structDecoder, buf []byte, cursor int64) (int64, *structFieldSet, error) { @@ -242,7 +246,10 @@ func decodeKeyByBitmapUint8(d *structDecoder, buf []byte, cursor int64) (int64, return 0, nil, errors.ErrUnexpectedEndOfJSON("string", cursor) case '\\': cursor++ - chars, nextCursor := decodeKeyCharByEscapedChar(buf, cursor) + chars, nextCursor, err := decodeKeyCharByEscapedChar(buf, cursor) + if err != nil { + return 0, nil, err + } for _, c := range chars { curBit &= bitmap[keyIdx][largeToSmallTable[c]] if curBit == 0 { @@ -305,7 +312,10 @@ func decodeKeyByBitmapUint16(d *structDecoder, buf []byte, cursor int64) (int64, return 0, nil, errors.ErrUnexpectedEndOfJSON("string", cursor) case '\\': cursor++ - chars, nextCursor := decodeKeyCharByEscapedChar(buf, cursor) + chars, nextCursor, err := decodeKeyCharByEscapedChar(buf, cursor) + if err != nil { + return 0, nil, err + } for _, c := range chars { curBit &= bitmap[keyIdx][largeToSmallTable[c]] if curBit == 0 { From 06ab2b4c8831e676ce45b84683aa735e173ba043 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Fri, 24 Feb 2023 09:13:03 +0900 Subject: [PATCH 4/9] fix: fixed handling of anonymous fields other than struct fix #426 --- encode_test.go | 15 +++++++++++++++ internal/encoder/compiler.go | 9 ++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/encode_test.go b/encode_test.go index 63740f35..3ff3fcb8 100644 --- a/encode_test.go +++ b/encode_test.go @@ -2629,3 +2629,18 @@ func TestCustomMarshalForMapKey(t *testing.T) { assertErr(t, err) assertEq(t, "custom map key", string(expected), string(got)) } + +func TestIssue426(t *testing.T) { + type I interface { + Foo() + } + type A struct { + I + Val string + } + var s A + s.Val = "456" + + b, _ := json.Marshal(s) + assertEq(t, "unexpected result", `{"I":null,"Val":"456"}`, string(b)) +} diff --git a/internal/encoder/compiler.go b/internal/encoder/compiler.go index 3b3ff3fd..3ae39ba8 100644 --- a/internal/encoder/compiler.go +++ b/internal/encoder/compiler.go @@ -617,6 +617,13 @@ func (c *Compiler) structCode(typ *runtime.Type, isPtr bool) (*StructCode, error return code, nil } +func toElemType(t *runtime.Type) *runtime.Type { + for t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t +} + func (c *Compiler) structFieldCode(structCode *StructCode, tag *runtime.StructTag, isPtr, isOnlyOneFirstField bool) (*StructFieldCode, error) { field := tag.Field fieldType := runtime.Type2RType(field.Type) @@ -626,7 +633,7 @@ func (c *Compiler) structFieldCode(structCode *StructCode, tag *runtime.StructTa key: tag.Key, tag: tag, offset: field.Offset, - isAnonymous: field.Anonymous && !tag.IsTaggedKey, + isAnonymous: field.Anonymous && !tag.IsTaggedKey && toElemType(fieldType).Kind() == reflect.Struct, isTaggedKey: tag.IsTaggedKey, isNilableType: c.isNilableType(fieldType), isNilCheck: true, From 2ef15e72f8df8a9c275459f4b1b335ccf768bbe3 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Mon, 13 Mar 2023 19:57:24 +0900 Subject: [PATCH 5/9] fix: fixed to not optimize when lower can't handle byte-by-byte. (#432) --- decode_test.go | 12 ++++++++++++ internal/decoder/struct.go | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/decode_test.go b/decode_test.go index a1a10e9c..9306b2e1 100644 --- a/decode_test.go +++ b/decode_test.go @@ -4016,6 +4016,18 @@ func TestIssue408(t *testing.T) { } } +func TestIssue416(t *testing.T) { + b := []byte(`{"Сообщение":"Текст"}`) + + type T struct { + Msg string `json:"Сообщение"` + } + var x T + err := json.Unmarshal(b, &x) + assertErr(t, err) + assertEq(t, "unexpected result", "Текст", x.Msg) +} + func TestIssue429(t *testing.T) { var x struct { N int32 diff --git a/internal/decoder/struct.go b/internal/decoder/struct.go index 4e14a20f..313da153 100644 --- a/internal/decoder/struct.go +++ b/internal/decoder/struct.go @@ -51,6 +51,14 @@ func init() { } } +func toASCIILower(s string) string { + b := []byte(s) + for i := range b { + b[i] = largeToSmallTable[b[i]] + } + return string(b) +} + func newStructDecoder(structName, fieldName string, fieldMap map[string]*structFieldSet) *structDecoder { return &structDecoder{ fieldMap: fieldMap, @@ -91,6 +99,10 @@ func (d *structDecoder) tryOptimize() { for k, v := range d.fieldMap { key := strings.ToLower(k) if key != k { + if key != toASCIILower(k) { + d.isTriedOptimize = true + return + } // already exists same key (e.g. Hello and HELLO has same lower case key if _, exists := conflicted[key]; exists { d.isTriedOptimize = true From f32a307cafe0b5a5fe6780589b4869e4d4af4123 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Mon, 13 Mar 2023 19:58:11 +0900 Subject: [PATCH 6/9] fix: fixed a problem that MarshalIndent does not work when UnorderedMap is specified (#435) --- encode_test.go | 16 ++++++++++++++++ internal/encoder/vm_color_indent/util.go | 7 ++++--- internal/encoder/vm_indent/util.go | 7 ++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/encode_test.go b/encode_test.go index 3ff3fcb8..f3b57bf3 100644 --- a/encode_test.go +++ b/encode_test.go @@ -2630,6 +2630,22 @@ func TestCustomMarshalForMapKey(t *testing.T) { assertEq(t, "custom map key", string(expected), string(got)) } +func TestIssue417(t *testing.T) { + x := map[string]string{ + "b": "b", + "a": "a", + } + b, err := json.MarshalIndentWithOption(x, "", " ", json.UnorderedMap()) + assertErr(t, err) + + var y map[string]string + err = json.Unmarshal(b, &y) + assertErr(t, err) + + assertEq(t, "key b", "b", y["b"]) + assertEq(t, "key a", "a", y["a"]) +} + func TestIssue426(t *testing.T) { type I interface { Foo() diff --git a/internal/encoder/vm_color_indent/util.go b/internal/encoder/vm_color_indent/util.go index 60e4a8ed..2395abec 100644 --- a/internal/encoder/vm_color_indent/util.go +++ b/internal/encoder/vm_color_indent/util.go @@ -189,7 +189,7 @@ func appendNullComma(ctx *encoder.RuntimeContext, b []byte) []byte { } func appendColon(_ *encoder.RuntimeContext, b []byte) []byte { - return append(b, ':', ' ') + return append(b[:len(b)-2], ':', ' ') } func appendMapKeyValue(ctx *encoder.RuntimeContext, code *encoder.Opcode, b, key, value []byte) []byte { @@ -229,8 +229,9 @@ func appendEmptyObject(_ *encoder.RuntimeContext, b []byte) []byte { func appendObjectEnd(ctx *encoder.RuntimeContext, code *encoder.Opcode, b []byte) []byte { last := len(b) - 1 - b[last] = '\n' - b = appendIndent(ctx, b, code.Indent-1) + // replace comma to newline + b[last-1] = '\n' + b = appendIndent(ctx, b[:last], code.Indent) return append(b, '}', ',', '\n') } diff --git a/internal/encoder/vm_indent/util.go b/internal/encoder/vm_indent/util.go index fca8f185..6cb745e3 100644 --- a/internal/encoder/vm_indent/util.go +++ b/internal/encoder/vm_indent/util.go @@ -133,7 +133,7 @@ func appendNullComma(_ *encoder.RuntimeContext, b []byte) []byte { } func appendColon(_ *encoder.RuntimeContext, b []byte) []byte { - return append(b, ':', ' ') + return append(b[:len(b)-2], ':', ' ') } func appendMapKeyValue(ctx *encoder.RuntimeContext, code *encoder.Opcode, b, key, value []byte) []byte { @@ -173,8 +173,9 @@ func appendEmptyObject(_ *encoder.RuntimeContext, b []byte) []byte { func appendObjectEnd(ctx *encoder.RuntimeContext, code *encoder.Opcode, b []byte) []byte { last := len(b) - 1 - b[last] = '\n' - b = appendIndent(ctx, b, code.Indent-1) + // replace comma to newline + b[last-1] = '\n' + b = appendIndent(ctx, b[:last], code.Indent) return append(b, '}', ',', '\n') } From 7be58ac89df93e099b5a212ae588f47b653dbf07 Mon Sep 17 00:00:00 2001 From: Anders Brander Date: Mon, 13 Mar 2023 12:01:01 +0100 Subject: [PATCH 7/9] Fix mapDecoder.DecodeStream() to accept empty objects containing whitespace (#425) --- decode_test.go | 8 ++++++++ internal/decoder/map.go | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/decode_test.go b/decode_test.go index 9306b2e1..df914eda 100644 --- a/decode_test.go +++ b/decode_test.go @@ -282,6 +282,14 @@ func Test_Decoder_DisallowUnknownFields(t *testing.T) { } } +func Test_Decoder_EmptyObjectWithSpace(t *testing.T) { + dec := json.NewDecoder(strings.NewReader(`{"obj":{ }}`)) + var v struct { + Obj map[string]int `json:"obj"` + } + assertErr(t, dec.Decode(&v)) +} + type unmarshalJSON struct { v int } diff --git a/internal/decoder/map.go b/internal/decoder/map.go index 7a6eea34..07a9caea 100644 --- a/internal/decoder/map.go +++ b/internal/decoder/map.go @@ -88,7 +88,7 @@ func (d *mapDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) erro mapValue = makemap(d.mapType, 0) } s.cursor++ - if s.equalChar('}') { + if s.skipWhiteSpace() == '}' { *(*unsafe.Pointer)(p) = mapValue s.cursor++ return nil From 6f969b6d5fa7a435ddab1f28af28ae8019747534 Mon Sep 17 00:00:00 2001 From: Nao Yonashiro Date: Mon, 13 Mar 2023 20:54:51 +0900 Subject: [PATCH 8/9] fix: fixed an issue that could not set the correct NextField for fields in the embedded structure (#438) fix #391 --- encode_test.go | 34 ++++++++++++++++++++++++++++++++++ internal/encoder/code.go | 20 +++++++++++++------- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/encode_test.go b/encode_test.go index f3b57bf3..d328c025 100644 --- a/encode_test.go +++ b/encode_test.go @@ -2630,6 +2630,40 @@ func TestCustomMarshalForMapKey(t *testing.T) { assertEq(t, "custom map key", string(expected), string(got)) } +func TestIssue391(t *testing.T) { + type A struct { + X string `json:"x,omitempty"` + } + type B struct { + A + } + type C struct { + X []int `json:"x,omitempty"` + } + for _, tc := range []struct { + name string + in interface{} + out string + }{ + {in: struct{ B }{}, out: "{}"}, + {in: struct { + B + Y string `json:"y"` + }{}, out: `{"y":""}`}, + {in: struct { + Y string `json:"y"` + B + }{}, out: `{"y":""}`}, + {in: struct{ C }{}, out: "{}"}, + } { + t.Run(tc.name, func(t *testing.T) { + b, err := json.Marshal(tc.in) + assertErr(t, err) + assertEq(t, "unexpected result", tc.out, string(b)) + }) + } +} + func TestIssue417(t *testing.T) { x := map[string]string{ "b": "b", diff --git a/internal/encoder/code.go b/internal/encoder/code.go index 8d62a9cd..66425a86 100644 --- a/internal/encoder/code.go +++ b/internal/encoder/code.go @@ -397,7 +397,10 @@ func (c *StructCode) lastFieldCode(field *StructFieldCode, firstField *Opcode) * func (c *StructCode) lastAnonymousFieldCode(firstField *Opcode) *Opcode { // firstField is special StructHead operation for anonymous structure. // So, StructHead's next operation is truly struct head operation. - lastField := firstField.Next + for firstField.Op == OpStructHead { + firstField = firstField.Next + } + lastField := firstField for lastField.NextField != nil { lastField = lastField.NextField } @@ -437,11 +440,6 @@ func (c *StructCode) ToOpcode(ctx *compileContext) Opcodes { } if isEndField { endField := fieldCodes.Last() - if isEmbeddedStruct(field) { - firstField.End = endField - lastField := c.lastAnonymousFieldCode(firstField) - lastField.NextField = endField - } if len(codes) > 0 { codes.First().End = endField } else { @@ -698,7 +696,15 @@ func (c *StructFieldCode) addStructEndCode(ctx *compileContext, codes Opcodes) O Indent: ctx.indent, } codes.Last().Next = end - codes.First().NextField = end + code := codes.First() + for code.Op == OpStructField || code.Op == OpStructHead { + code = code.Next + } + for code.NextField != nil { + code = code.NextField + } + code.NextField = end + codes = codes.Add(end) ctx.incOpcodeIndex() return codes From fbd4feeb6079b0ee46450accc931bbc379671d6d Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Mon, 13 Mar 2023 21:04:33 +0900 Subject: [PATCH 9/9] Update CHANGELOG --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9941520..909b971e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# v0.10.1 - 2023/03/13 + +### Fix bugs + +* Fix checkptr error for array decoder ( #415 ) +* Fix added buffer size check when decoding key ( #430 ) +* Fix handling of anonymous fields other than struct ( #431 ) +* Fix to not optimize when lower conversion can't handle byte-by-byte ( #432 ) +* Fix a problem that MarshalIndent does not work when UnorderedMap is specified ( #435 ) +* Fix mapDecoder.DecodeStream() for empty objects containing whitespace ( #425 ) +* Fix an issue that could not set the correct NextField for fields in the embedded structure ( #438 ) + # v0.10.0 - 2022/11/29 ### New features