fix(arm): fix out of bound in native decoder

bytedance · Jul 24, 2024 · 4c21849 · 4c21849
1 parent 15dff36
commit 4c21849
Show file tree

Hide file tree

Showing 12 changed files with 34,399 additions and 34,552 deletions.
diff --git a/internal/decoder/optdec/native.go b/internal/decoder/optdec/native.go
@@ -1,6 +1,7 @@
 package optdec
 
 import (
+	"fmt"
 	"reflect"
 	"unsafe"
 
@@ -174,8 +175,12 @@ func (p *Parser) JsonBytes() []byte {
 
 var nodeType = rt.UnpackType(reflect.TypeOf(node{}))
 
+//go:inline
+func calMaxNodeCap(jsonSize int) int {
+	return jsonSize / 2 + 2
+}
+
 func (p *Parser) parse() ErrorCode {
-	var offset uintptr
 	// when decode into struct, we should decode number as possible
 	old := p.options
 	if !p.isEface {
@@ -189,15 +194,22 @@ func (p *Parser) parse() ErrorCode {
 		return err
 	}
 
+	// check OoB here
+	println("retyru origin json is ", p.Json)
+	offset := p.nbuf.ncur - p.nbuf.nstart
+	curLen :=  offset / unsafe.Sizeof(node{})
+	if curLen != uintptr(len(p.nodes)) {
+		panic(fmt.Sprintf("current len: %d, real len: %d cap: %d", curLen, len(p.nodes), cap(p.nodes)))
+	}
+
 	// node buf is not enough, continue parse
 	// the maxCap is always meet all valid JSON
-	maxCap := len(p.Json) / 2 + 2 
+	maxCap := calMaxNodeCap(len(p.Json))
 	slice := rt.GoSlice{
 		Ptr: rt.Mallocgc(uintptr(maxCap) * nodeType.Size, nodeType, false),
 		Len: maxCap,
 		Cap: maxCap,
 	}
-	offset = p.nbuf.ncur - p.nbuf.nstart
 	rt.Memmove(unsafe.Pointer(slice.Ptr), unsafe.Pointer(&p.nodes[0]), offset)
 	p.backup = p.nodes
 	p.nodes = *(*[]node)(unsafe.Pointer(&slice))

diff --git a/internal/decoder/optdec/native_test.go b/internal/decoder/optdec/native_test.go
@@ -32,6 +32,18 @@ func TestParseNativeRetryLargeJson(t *testing.T) {
 		assert.Equal(t, int(p.nbuf.stat.number), 1 << 20 + 1)
 	})
 
+	t.Run("ObjectNull", func (t *testing.T)  {
+		data := "{" + strings.Repeat("\"a\":null,", 1 << 20) + "\"a\":null}"
+		p := newParser(data, 0, 0)
+		ecode := p.parse()
+		defer p.free()
+		assert.Equal(t, int(ecode), 0)
+		assert.Equal(t, int(p.Pos()), len(data))
+		assert.Equal(t, int(p.nbuf.stat.object),  1)
+		assert.Equal(t, int(p.nbuf.stat.object_keys), 1 << 20 + 1)
+		assert.Equal(t, int(p.nbuf.stat.max_depth), 1)
+	})
+
 	t.Run("Object2", func (t *testing.T)  {
 		data := "{\"top\": {" + strings.Repeat("\"a\":1,", 1 << 20) + "\"a\":1}, \"final\": true}"
 		p := newParser(data, 0, 0)

diff --git a/internal/native/avx2/parse_with_padding_subr.go b/internal/native/avx2/parse_with_padding_subr.go
diff --git a/internal/native/avx2/parse_with_padding_text_amd64.go b/internal/native/avx2/parse_with_padding_text_amd64.go
diff --git a/internal/native/neon/f32toa_arm64.s b/internal/native/neon/f32toa_arm64.s
@@ -916,7 +916,7 @@ _Digits:
 	WORD $0x37393639  // .ascii 4, '96979899'
 	WORD $0x39393839  // .ascii 4, '9899'
 	WORD $0x00000000  // .p2align 3, 0x00
-_LB_abe163c3: // _pow10_ceil_sig_f32.g
+_LB_0c2486d9: // _pow10_ceil_sig_f32.g
 	WORD $0x4b43fcf5; WORD $0x81ceb32c  // .quad -9093133594791772939
 	WORD $0x5e14fc32; WORD $0xa2425ff7  // .quad -6754730975062328270
 	WORD $0x359a3b3f; WORD $0xcad2f7f5  // .quad -3831727700400522433

diff --git a/internal/native/neon/f64toa_arm64.s b/internal/native/neon/f64toa_arm64.s
@@ -1232,7 +1232,7 @@ _Digits:
 	WORD $0x37393639  // .ascii 4, '96979899'
 	WORD $0x39393839  // .ascii 4, '9899'
 	  // .p2align 3, 0x00
-_LB_a35fcfcc: // _pow10_ceil_sig.g
+_LB_fe6a4ef9: // _pow10_ceil_sig.g
 	WORD $0xbebcdc4f; WORD $0xff77b1fc  // .quad -38366372719436721
 	WORD $0x13bb0f7b; WORD $0x25e8e89c  // .quad 2731688931043774331
 	WORD $0xf73609b1; WORD $0x9faacf3d  // .quad -6941508010590729807