feat: support aarch64 (#645)

Co-authored-by: duanyi.aster <[email protected]>
bytedance · Jul 8, 2024 · 765ecdb · 765ecdb
1 parent f16c69f
commit 765ecdb
Show file tree

Hide file tree

Showing 212 changed files with 60,323 additions and 9,271 deletions.
diff --git a/.github/workflows/benchmark-linux-x64.yml → .github/workflows/benchmark-linux.yml b/.github/workflows/benchmark-linux-x64.yml → .github/workflows/benchmark-linux.yml
@@ -4,7 +4,10 @@ on: pull_request
 
 jobs:
   build:
-    runs-on: [self-hosted, X64]
+    strategy:
+      matrix:
+        os: [X64, arm]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Clear repository
         run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE

diff --git a/.github/workflows/compatibility_test.yml b/.github/workflows/compatibility_test.yml
@@ -6,7 +6,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
+        go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
         os: [arm, X64]
     runs-on: ${{ matrix.os }}
     steps:
@@ -28,10 +28,16 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: main
-        run: GOMAXPROCS=4 go test -v -gcflags="all=-l" -race github.com/bytedance/sonic
+        run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic
+
+      - name: decoder
+        run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/decoder
+
+      - name: encoder
+        run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/encoder
 
       - name: ast
-        run: GOMAXPROCS=4 go test -v -gcflags="all=-l" -race github.com/bytedance/sonic/ast
+        run: go test -race -v -gcflags="all=-l" github.com/bytedance/sonic/ast
 
       - name: qemu
         run: sh scripts/qemu.sh
diff --git a/.github/workflows/fuzzing-linux-opt-X64.yml b/.github/workflows/fuzzing-linux-opt-X64.yml
@@ -0,0 +1,33 @@
+name: Fuzz Test Linux-X64
+
+on: pull_request
+
+jobs:
+  build:
+    strategy:
+        matrix:
+          os: [arm, X64]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.20.x
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Fuzz sonic
+        run: |
+          cd ./fuzz
+          make fuzz
+          make runopt 
diff --git a/.github/workflows/fuzzing-linux-x64.yml b/.github/workflows/fuzzing-linux-x64.yml
@@ -4,7 +4,10 @@ on: pull_request
 
 jobs:
   build:
-    runs-on: [self-hosted, X64]
+    strategy:
+        matrix:
+          os: [arm, X64]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Clear repository
         run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
@@ -14,7 +17,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.20.x
 
       - uses: actions/cache@v2
         with:

diff --git a/.github/workflows/benchmark-linux-arm64.yml → .github/workflows/generic_test-opt.yml b/.github/workflows/benchmark-linux-arm64.yml → .github/workflows/generic_test-opt.yml
@@ -1,10 +1,10 @@
-name: Benchmark Linux-ARM
+name: Generic Test Go1.18-Linux-X64
 
 on: pull_request
 
 jobs:
   build:
-    runs-on: [arm]
+    runs-on: [self-hosted, X64]
     steps:
       - name: Clear repository
         run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.22
+          go-version: 1.18
 
       - uses: actions/cache@v2
         with:
@@ -23,5 +23,5 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
-      - name: Benchmark sonic
-        run: sh scripts/bench-arm.sh
+      - name: Generic Test
+        run: GOMAXPROCS=4 SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -v -race ./generic_test
diff --git a/.github/workflows/unit_test-linux-arm.yml b/.github/workflows/unit_test-linux-arm.yml
@@ -0,0 +1,42 @@
+name: Unit Test Linux arm
+
+on: push
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        # TODO: 1.17.x, 1.18.x, 1.19.x not supported because golang asm bug
+        go-version: [1.20.x, 1.21.x, 1.22.x]
+    runs-on: [arm]
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Unit Test
+        run: |
+          go test -race -covermode=atomic -coverprofile=coverage.txt  $(go list ./... | grep -v -E 'loader|jit|avx|x86|sse')
+
+      - name: external
+        run: |
+          cd ./external_jsonlib_test
+          GOMAXPROCS=4 go test -v -race ./...
+      
+      - name: Codecov
+        run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/unit_test-linux-opt-x64.yml b/.github/workflows/unit_test-linux-opt-x64.yml
@@ -0,0 +1,46 @@
+name: Unit Test Linux X64
+
+on: push
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        go-version: [1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
+    runs-on: [self-hosted, X64]
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Unit Test
+        run: |
+          GOMAXPROCS=4 SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -race -covermode=atomic -coverprofile=coverage.txt ./...
+
+      - name: external
+        run: |
+          cd ./external_jsonlib_test
+          SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -v -race ./...
+
+      - name: external
+        run: |
+          cd ./loader
+          SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -v -race ./...
+      
+      - name: Codecov
+        run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/unit_test-linux-x64.yml b/.github/workflows/unit_test-linux-x64.yml
@@ -1,4 +1,4 @@
-name: Unit Test Linux-X64
+name: Unit Test Linux X64
 
 on: push
 
@@ -7,7 +7,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        go-version: [1.16.x, 1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
+        go-version: [1.17.x, 1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x]
     runs-on: [self-hosted, X64]
     steps:
       - name: Clear repository
@@ -40,17 +40,17 @@ jobs:
 
       - name: Unit Test
         run: |
-          go test -race -covermode=atomic -coverprofile=coverage.txt ./...
+          GOMAXPROCS=4 go test -race -covermode=atomic -coverprofile=coverage.txt ./...
 
       - name: external
         run: |
           cd ./external_jsonlib_test
-          GOMAXPROCS=4 go test -v -race ./...
+          go test -v -race ./...
 
       - name: external
         run: |
           cd ./loader
-          GOMAXPROCS=4 go test -v -race ./...
+          go test -v -race ./...
       
       - name: Codecov
         run: bash <(curl -s https://codecov.io/bash)
diff --git a/.gitignore b/.gitignore
@@ -49,7 +49,6 @@ ast/bench.sh
 
 !testdata/*.json.gz
 fuzz/testdata
-*__debug_bin
-
-*.pprof
+*__debug_bin*
+*pprof
 *coverage.txt
diff --git a/ast/api.go b/ast/api.go
@@ -1,5 +1,5 @@
-//go:build (amd64 && go1.16 && !go1.23) || (arm64 && go1.20 && !go1.23)
-// +build amd64,go1.16,!go1.23 arm64,go1.20,!go1.23
+//go:build (amd64 && go1.17 && !go1.23) || (arm64 && go1.20 && !go1.23)
+// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23
 
 /*
  * Copyright 2022 ByteDance Inc.

diff --git a/ast/api_compat.go b/ast/api_compat.go
@@ -1,4 +1,4 @@
-// +build !amd64,!arm64 go1.23 !go1.16 arm64,!go1.20
+// +build !amd64,!arm64 go1.23 !go1.17 arm64,!go1.20
 
 /*
 * Copyright 2022 ByteDance Inc.
@@ -27,7 +27,7 @@ import (
 )
 
 func init() {
-    println("WARNING:(ast) sonic only supports Go1.16~1.22, but your environment is not suitable")
+    println("WARNING:(ast) sonic only supports go1.17~1.22, but your environment is not suitable")
 }
 
 func quote(buf *[]byte, val string) {

diff --git a/ast/api_native_test.go b/ast/api_native_test.go
@@ -1,5 +1,5 @@
-//go:build (amd64 && go1.16 && !go1.23) || (arm64 && go1.20 && !go1.23)
-// +build amd64,go1.16,!go1.23 arm64,go1.20,!go1.23
+//go:build (amd64 && go1.17 && !go1.23) || (arm64 && go1.20 && !go1.23)
+// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23
 
 /*
  * Copyright 2022 ByteDance Inc.

diff --git a/ast/b64_amd64.go b/ast/b64_amd64.go
@@ -1,4 +1,4 @@
-// +build amd64,go1.16
+// +build amd64,go1.17
 
 /**
  * Copyright 2023 ByteDance Inc.

diff --git a/ast/b64_compat.go b/ast/b64_compat.go
@@ -1,4 +1,4 @@
-// +build !amd64 !go1.16
+// +build !amd64 !go1.17
 
 /*
  * Copyright 2022 ByteDance Inc.

diff --git a/compat.go b/compat.go
@@ -1,4 +1,4 @@
-// +build !amd64 !go1.16 go1.23
+// +build !amd64,!arm64 go1.23 !go1.17 arm64,!go1.20
 
 /*
  * Copyright 2021 ByteDance Inc.

diff --git a/decode_test.go b/decode_test.go
@@ -1,4 +1,5 @@
-// +build amd64,go1.16,!go1.23
+//go:build (amd64 && go1.17 && !go1.23) || (arm64 && go1.20 && !go1.23)
+// +build amd64,go1.17,!go1.23 arm64,go1.20,!go1.23
 
 /*
  * Copyright 2021 ByteDance Inc.
@@ -458,6 +459,8 @@ var unmarshalTests = []unmarshalTest{
     {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: json.Number("1"), F2: int32(2), F3: json.Number("3")}, useNumber: true},
     {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64},
     {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true},
+	{in: `{"":""}`, ptr: new(struct{}), out: struct{}{}},
+	{in: `{"x":""}`, ptr: new(struct{ X json.Number }),  err: errors.New("empty string into json number")},
 
     // raw values with whitespace
     {in: "\n true ", ptr: new(bool), out: true},
@@ -1116,7 +1119,6 @@ func TestMarshalEmbeds(t *testing.T) {
 
 func TestUnmarshal(t *testing.T) {
     for i, tt := range unmarshalTests {
-        t.Log(i, tt.in)
         if !json.Valid([]byte(tt.in)) {
             continue
         }

diff --git a/decoder/decoder_compat.go b/decoder/decoder_compat.go
@@ -1,4 +1,4 @@
-// +build !amd64 !go1.16 go1.23
+// +build !amd64,!arm64 go1.23 !go1.17 arm64,!go1.20
 
 /*
 * Copyright 2023 ByteDance Inc.
@@ -30,7 +30,7 @@ import (
 )
 
 func init() {
-     println("WARNING: sonic only supports Go1.16~1.22 && CPU amd64, but your environment is not suitable")
+     println("WARNING: sonic/decoder only supports (Go1.17~1.22 && CPU amd64) or (go1.20~1.22 && CPU arm64), but your environment is not suitable")
 }
 
 const (