opt: mitigate big map issue of connections when garbage collecting (#460

) goos: darwin goarch: arm64 pkg: github.com/panjf2000/gnet/v2 │ old │ new │ │ sec/op │ sec/op vs base │ GC4El100k/Run-4-eventloop-100000-10 30.74m ± 3% 19.68m ± 10% -35.98% (p=0.000 n=10) GC4El200k/Run-4-eventloop-200000-10 63.64m ± 3% 38.16m ± 11% -40.04% (p=0.000 n=10) GC4El500k/Run-4-eventloop-500000-10 177.28m ± 8% 95.21m ± 4% -46.29% (p=0.000 n=10) geomean 70.26m 41.51m -40.92% │ old │ new │ │ B/op │ B/op vs base │ GC4El100k/Run-4-eventloop-100000-10 27.50 ± 35% 25.50 ± 33% ~ (p=0.423 n=10) GC4El200k/Run-4-eventloop-200000-10 27.50 ± 53% 20.50 ± 66% ~ (p=0.642 n=10) GC4El500k/Run-4-eventloop-500000-10 16.00 ± ? 18.00 ± ? ~ (p=0.357 n=10) geomean 22.96 21.11 -8.04% │ old │ new │ │ allocs/op │ allocs/op vs base │ GC4El100k/Run-4-eventloop-100000-10 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ GC4El200k/Run-4-eventloop-200000-10 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ GC4El500k/Run-4-eventloop-500000-10 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ geomean ² +0.00% ² ¹ all samples are equal ² summaries must be >0 to compute geomean Fixes #334 --------- Co-authored-by: Andy Pan <[email protected]>
panjf2000 · May 16, 2023 · dd48bac · dd48bac
1 parent 94412fb
commit dd48bac
Show file tree

Hide file tree

Showing 27 changed files with 1,475 additions and 248 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -42,7 +42,7 @@ jobs:
       - name: Setup and run golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.51.2
+          version: v1.52.2
           args: -v -E gofumpt -E gocritic -E misspell -E revive -E godot --timeout 5m
   test:
     needs: lint
@@ -94,7 +94,7 @@ jobs:
         run: go test $(go list ./... | tail -n +2)
 
       - name: Run integration tests
-        run: go test -v -race -coverprofile="codecov.report" -covermode=atomic -timeout 5m
+        run: go test -v -race -coverprofile="codecov.report" -covermode=atomic -timeout 10m -failfast
 
       - name: Upload the code coverage report to codecov.io
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/test_gc_opt.yml b/.github/workflows/test_gc_opt.yml
@@ -0,0 +1,106 @@
+name: Run tests with -tags=gc_opt
+
+on:
+  push:
+    branches:
+      - master
+      - dev
+      - 1.x
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches:
+      - master
+      - dev
+      - 1.x
+    paths-ignore:
+      - '**.md'
+
+env:
+  GO111MODULE: on
+  GOPROXY: "https://proxy.golang.org"
+
+jobs:
+  lint:
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          #- windows-latest
+    name: Run golangci-lint
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '^1.17'
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup and run golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.52.2
+          args: -v -E gofumpt -E gocritic -E misspell -E revive -E godot --timeout 5m
+  test:
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ['1.17', '1.18', '1.19', '1.20']
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    name: Go ${{ matrix.go }} @ ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Print Go environment
+        id: go-env
+        run: |
+          printf "Using go at: $(which go)\n"
+          printf "Go version: $(go version)\n"
+          printf "\n\nGo environment:\n\n"
+          go env
+          printf "\n\nSystem environment:\n\n"
+          env
+          # Calculate the short SHA1 hash of the git commit
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "GO_CACHE=$(go env GOCACHE)" >> $GITHUB_OUTPUT
+
+      - name: Cache go modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ steps.go-env.outputs.GO_CACHE }}
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go }}-go-ci-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go }}-go-ci
+
+      - name: Run unit tests for packages
+        run: go test $(go list ./... | tail -n +2)
+
+      - name: Run integration tests
+        run: go test -v -race -tags=gc_opt -coverprofile="codecov.report" -covermode=atomic -timeout 10m -failfast
+
+      - name: Upload the code coverage report to codecov.io
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./codecov.report
+          flags: unittests
+          name: codecov-gnet
+          fail_ci_if_error: true
+          verbose: true
diff --git a/.github/workflows/test_poll_opt.yml b/.github/workflows/test_poll_opt.yml
@@ -41,7 +41,7 @@ jobs:
       - name: Setup and run golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.51.2
+          version: v1.52.2
           args: -v -E gofumpt -E gocritic -E misspell -E revive -E godot
   test:
     needs: lint
@@ -90,7 +90,7 @@ jobs:
         run: go test $(go list ./... | tail -n +2)
 
       - name: Run integration tests with -tags=poll_opt
-        run: go test -v -tags=poll_opt -coverprofile="codecov.report" -covermode=atomic -timeout 5m
+        run: go test -v -tags=poll_opt -coverprofile="codecov.report" -covermode=atomic -timeout 10m -failfast
 
       - name: Upload the code coverage report to codecov.io
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/test_poll_opt_gc_opt.yml b/.github/workflows/test_poll_opt_gc_opt.yml
@@ -0,0 +1,102 @@
+name: Run tests with -tags=poll_opt,gc_opt
+
+on:
+  push:
+    branches:
+      - master
+      - dev
+      - 1.x
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches:
+      - master
+      - dev
+      - 1.x
+    paths-ignore:
+      - '**.md'
+
+env:
+  GO111MODULE: on
+  GOPROXY: "https://proxy.golang.org"
+
+jobs:
+  lint:
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+    name: Run golangci-lint
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '^1.17'
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup and run golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.52.2
+          args: -v -E gofumpt -E gocritic -E misspell -E revive -E godot
+  test:
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        go: ['1.17', '1.18', '1.19', '1.20']
+        os: [ubuntu-latest, macos-latest]
+    name: Go ${{ matrix.go }} @ ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '^1.17'
+
+      - name: Print Go environment
+        id: go-env
+        run: |
+          printf "Using go at: $(which go)\n"
+          printf "Go version: $(go version)\n"
+          printf "\n\nGo environment:\n\n"
+          go env
+          printf "\n\nSystem environment:\n\n"
+          env
+          # Calculate the short SHA1 hash of the git commit
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          echo "GO_CACHE=$(go env GOCACHE)" >> $GITHUB_OUTPUT
+
+      - name: Cache go modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ steps.go-env.outputs.GO_CACHE }}
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go }}-go-ci-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go }}-go-ci
+
+      - name: Run unit tests for packages
+        run: go test $(go list ./... | tail -n +2)
+
+      - name: Run integration tests with -tags=poll_opt
+        run: go test -v -tags=poll_opt,gc_opt -coverprofile="codecov.report" -covermode=atomic -timeout 10m -failfast
+
+      - name: Upload the code coverage report to codecov.io
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./codecov.report
+          flags: unittests
+          name: codecov-gnet-poll_opt
+          fail_ci_if_error: true
+          verbose: true
diff --git a/acceptor_unix.go b/acceptor_unix.go
@@ -59,9 +59,8 @@ func (eng *engine) accept(fd int, _ netpoll.IOEvent) error {
 	if err != nil {
 		eng.opts.Logger.Errorf("UrgentTrigger() failed due to error: %v", err)
 		_ = unix.Close(nfd)
-		c.releaseTCP()
+		c.release()
 	}
-
 	return nil
 }
 
@@ -94,10 +93,9 @@ func (el *eventloop) accept(fd int, ev netpoll.IOEvent) error {
 	}
 
 	c := newTCPConn(nfd, el, sa, el.ln.addr, remoteAddr)
-	if err = el.poller.AddRead(c.pollAttachment); err != nil {
+	if err = el.poller.AddRead(&c.pollAttachment); err != nil {
 		return err
 	}
-	el.connections[c.fd] = c
-
+	el.connections.addConn(c, el.idx)
 	return el.open(c)
 }
diff --git a/client_test.go b/client_test.go
@@ -47,7 +47,7 @@ func (ev *clientEvents) OnOpen(c Conn) ([]byte, Action) {
 	return nil, None
 }
 
-func (ev *clientEvents) OnClose(c Conn, err error) Action {
+func (ev *clientEvents) OnClose(Conn, error) Action {
 	if ev.svr != nil {
 		if atomic.AddInt32(&ev.svr.clientActive, -1) == 0 {
 			return Shutdown

diff --git a/client_unix.go b/client_unix.go
@@ -103,8 +103,7 @@ func NewClient(eh EventHandler, opts ...Option) (cli *Client, err error) {
 	}
 
 	el.buffer = make([]byte, options.ReadBufferCap)
-	el.udpSockets = make(map[int]*conn)
-	el.connections = make(map[int]*conn)
+	el.connections.init()
 	el.eventHandler = eh
 	cli.el = &el
 	return

diff --git a/conn_matrix.go b/conn_matrix.go
@@ -0,0 +1,73 @@
+// Copyright (c) 2023 Andy Pan.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build (linux || freebsd || dragonfly || darwin) && !gc_opt
+// +build linux freebsd dragonfly darwin
+// +build !gc_opt
+
+package gnet
+
+import (
+	"sync/atomic"
+
+	"github.com/panjf2000/gnet/v2/internal/gfd"
+)
+
+type connMatrix struct {
+	connCount int32
+	connMap   map[int]*conn
+}
+
+func (cm *connMatrix) init() {
+	cm.connMap = make(map[int]*conn)
+}
+
+func (cm *connMatrix) iterate(f func(*conn) bool) {
+	for _, c := range cm.connMap {
+		if c != nil {
+			if !f(c) {
+				return
+			}
+		}
+	}
+}
+
+func (cm *connMatrix) incCount(_ int, delta int32) {
+	atomic.AddInt32(&cm.connCount, delta)
+}
+
+func (cm *connMatrix) loadCount() (n int32) {
+	return atomic.LoadInt32(&cm.connCount)
+}
+
+func (cm *connMatrix) addConn(c *conn, index int) {
+	c.gfd = gfd.NewGFD(c.fd, index, 0, 0)
+	cm.connMap[c.fd] = c
+	cm.incCount(0, 1)
+}
+
+func (cm *connMatrix) delConn(c *conn) {
+	delete(cm.connMap, c.fd)
+	cm.incCount(0, -1)
+}
+
+func (cm *connMatrix) getConn(fd int) *conn {
+	return cm.connMap[fd]
+}
+
+/*
+func (cm *connMatrix) getConnByGFD(fd gfd.GFD) *conn {
+	return cm.connMap[fd.Fd()]
+}
+*/