[Bug]: TestCancelRunningTask failed

[qingxinhome]

Is there an existing issue for the same bug?

• I have checked the existing issues.

Branch Name

main

Commit ID

eeed8d9

Other Environment Information

- Hardware parameters:
- OS type:
- Others:

Actual Behavior

[OUTPUTS]:
=== RUN TestCancelRunningTask
2024/12/20 07:14:38.044693 +0000 DEBUG task-runner fetch task started {"runner-id": "r1"}
2024/12/20 07:14:38.045508 +0000 DEBUG task-runner dispatch task started {"runner-id": "r1"}
2024/12/20 07:14:38.046351 +0000 DEBUG task-runner done task started {"runner-id": "r1"}
2024/12/20 07:14:38.047133 +0000 DEBUG task-runner heartbeat task started {"runner-id": "r1"}
2024/12/20 07:14:38.048091 +0000 DEBUG task-runner retry task started {"runner-id": "r1"}
2024/12/20 07:14:38.048057 +0000 DEBUG task-runner executor registered {"runner-id": "r1", "code": "TestOnly"}
2024/12/20 07:14:38.049874 +0000 INFO task-runner new task fetched {"runner-id": "r1", "task": "t1/0"}
2024/12/20 07:14:38.049953 +0000 INFO task-runner task added to wait queue {"runner-id": "r1", "task": "t1/0"}
2024/12/20 07:14:38.050031 +0000 INFO task-runner task removed {"runner-id": "r1", "task-id": 1}
2024/12/20 07:14:38.050070 +0000 DEBUG task-runner task start execute {"runner-id": "r1", "task": "t1/0"}
2024/12/20 07:14:38.050081 +0000 ERROR task-runner task heartbeat failed {"runner-id": "r1", "task": "t1/0", "error": "invalid task, task runner r1, id 1"}
2024/12/20 07:14:38.050145 +0000 DEBUG task-runner task execute completed {"runner-id": "r1", "task": "t1/0", "cost": "94.294µs"}
2024/12/20 07:14:38.052406 +0000 INFO task-runner new task fetched {"runner-id": "r1", "task": "t1/0"}
2024/12/20 07:14:38.052518 +0000 INFO task-runner task added to wait queue {"runner-id": "r1", "task": "t1/0"}
2024/12/20 07:14:38.052680 +0000 DEBUG task-runner task start execute {"runner-id": "r1", "task": "t1/0"}
panic: test timed out after 40m0s
running tests:
TestCancelRunningTask (39m59s)

goroutine 307 [running]:
testing.(*M).startAlarm.func1()
/opt/hostedtoolcache/go/1.23.0/x64/src/testing/testing.go:2373 +0x265
created by time.goFunc
/opt/hostedtoolcache/go/1.23.0/x64/src/time/sleep.go:215 +0x45

goroutine 1 [chan receive, 39 minutes]:
testing.(*T).Run(0xc0000d64e0, {0x21ed815, 0x15}, 0x2258890)
/opt/hostedtoolcache/go/1.23.0/x64/src/testing/testing.go:1751 +0x851
testing.runTests.func1(0xc0000d64e0)
/opt/hostedtoolcache/go/1.23.0/x64/src/testing/testing.go:2168 +0x86
testing.tRunner(0xc0000d64e0, 0xc0010adae0)
/opt/hostedtoolcache/go/1.23.0/x64/src/testing/testing.go:1690 +0x227
testing.runTests(0xc0008569d8, {0x3166a80, 0x48, 0x48}, {0x7f0c3d3b0108?, 0x40?, 0x31ccc80?})
/opt/hostedtoolcache/go/1.23.0/x64/src/testing/testing.go:2166 +0x8bf
testing.(*M).Run(0xc0009621e0)
/opt/hostedtoolcache/go/1.23.0/x64/src/testing/testing.go:2034 +0xf18
main.main()
_testmain.go:187 +0x165

goroutine 10 [select]:
github.com/panjf2000/ants/v2.(*Pool).purgeStaleWorkers(0xc0004fb400, {0x246a8e0, 0xc0000e56d0})
/home/runner/go/pkg/mod/github.com/panjf2000/ants/[email protected]/pool.go:83 +0x1ef
created by github.com/panjf2000/ants/v2.(*Pool).goPurge in goroutine 1
/home/runner/go/pkg/mod/github.com/panjf2000/ants/[email protected]/pool.go:148 +0x185

goroutine 11 [select]:
github.com/panjf2000/ants/v2.(*Pool).ticktock(0xc0004fb400, {0x246a8e0, 0xc0000e5720})
/home/runner/go/pkg/mod/github.com/panjf2000/ants/[email protected]/pool.go:126 +0x1ec
created by github.com/panjf2000/ants/v2.(*Pool).goTicktock in goroutine 1
/home/runner/go/pkg/mod/github.com/panjf2000/ants/[email protected]/pool.go:155 +0x187

goroutine 281 [select, 39 minutes]:
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).dispatch(0xc002a1d340, {0x246a8a8, 0xc0032fc7e0})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner.go:420 +0x17c
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 279
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165

goroutine 324 [select, 39 minutes]:
github.com/matrixorigin/matrixone/pkg/taskservice.TestCancelRunningTask.func1.1({0x246a8e0, 0xc0000e43c0}, {0x12?, 0xc0012739c0?})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner_test.go:172 +0xa7
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).run.func1({0x246a8a8?, 0xc003311a40?})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner.go:486 +0x464
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 281
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165

goroutine 13 [chan receive]:
github.com/matrixorigin/matrixone/pkg/fileservice.init.2.func1()
/home/runner/work/matrixone/matrixone/pkg/fileservice/http_client.go:73 +0x86
created by github.com/matrixorigin/matrixone/pkg/fileservice.init.2 in goroutine 1
/home/runner/work/matrixone/matrixone/pkg/fileservice/http_client.go:72 +0x25

goroutine 286 [select, 39 minutes]:
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).handleTask(0xc002a1d340, {0x246a8a8, 0xc0032fc9c0})
/home/runner/work/matrixone/matrixone/pkg/taskservice/daemon_task.go:529 +0x116
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 279
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165

goroutine 284 [select]:
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).retry(0xc002a1d340, {0x246a8a8, 0xc0032fc900})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner.go:440 +0x1df
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 279
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165

goroutine 279 [sleep]:
time.Sleep(0x989680)
/opt/hostedtoolcache/go/1.23.0/x64/src/runtime/time.go:285 +0xf2
github.com/matrixorigin/matrixone/pkg/taskservice.TestCancelRunningTask.func1(0xc002a1d340, {0x249e3c0, 0xc003386420}, {0x249d018, 0xc0033861e0})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner_test.go:187 +0x5d1
github.com/matrixorigin/matrixone/pkg/taskservice.runTaskRunnerTest(0xc0000d6000, 0xc00225bea8, {0xc00225be90, 0x3, 0x3})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner_test.go:305 +0x85d
github.com/matrixorigin/matrixone/pkg/taskservice.TestCancelRunningTask(0xc0000d6000)
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner_test.go:169 +0x176
testing.tRunner(0xc0000d6000, 0x2258890)
/opt/hostedtoolcache/go/1.23.0/x64/src/testing/testing.go:1690 +0x227
created by testing.(*T).Run in goroutine 1
/opt/hostedtoolcache/go/1.23.0/x64/src/testing/testing.go:1743 +0x826

goroutine 282 [select, 39 minutes]:
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).done(0xc002a1d340, {0x246a8a8, 0xc0032fc840})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner.go:557 +0x17c
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 279
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165

goroutine 283 [select]:
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).heartbeat(0xc002a1d340, {0x246a8a8, 0xc0032fc8a0})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner.go:601 +0x1c5
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 279
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165

goroutine 285 [select]:
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).poll(0xc002a1d340, {0x246a8a8, 0xc0032fc960})
/home/runner/work/matrixone/matrixone/pkg/taskservice/daemon_task.go:309 +0x195
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 279
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165

goroutine 280 [select]:
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).fetch(0xc002a1d340, {0x246a8a8, 0xc0032fc780})
/home/runner/work/matrixone/matrixone/pkg/taskservice/task_runner.go:336 +0x1ef
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 279
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165

goroutine 287 [select]:
github.com/matrixorigin/matrixone/pkg/taskservice.(*taskRunner).sendHeartbeat(0xc002a1d340, {0x246a8a8, 0xc0032fca20})
/home/runner/work/matrixone/matrixone/pkg/taskservice/daemon_task.go:544 +0x189
github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask.func1()
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:277 +0xcf
created by github.com/matrixorigin/matrixone/pkg/common/stopper.(*Stopper).doRunCancelableTask in goroutine 279
/home/runner/work/matrixone/matrixone/pkg/common/stopper/stopper.go:272 +0x165
steps.ut.conclusion: failure
FAIL_UT_CASES=TestCancelRunningTask,

Expected Behavior

no failure

Steps to Reproduce

https://github.com/matrixorigin/matrixone/actions/runs/12427048369/job/34696265417?pr=20856

Additional information

No response