Skip to content

chore: better output on errors #2879

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 21, 2025
+18 −7
Merged

chore: better output on errors #2879

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
414fd21
Better log output for e2e
alpe Nov 21, 2025
ffbf876
Stacktrace on panic
alpe Nov 21, 2025
49f8963
Proper merge
alpe Nov 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions pkg/cmd/run_node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"
"os/signal"
"path/filepath"
"runtime"
"strings"
"syscall"
"time"
Expand Down Expand Up @@ -155,8 +156,10 @@ func StartNode(
go func() {
defer func() {
if r := recover(); r != nil {
err := fmt.Errorf("node panicked: %v", r)
logger.Error().Interface("panic", r).Msg("Recovered from panic in node")
buf := make([]byte, 1024)
Copy link
Contributor

@gemini-code-assist gemini-code-assist bot Nov 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The buffer size of 1024 bytes for capturing the stack trace might be too small for complex panics, potentially leading to truncated and incomplete stack traces. This would hinder debugging, which is contrary to the goal of this PR. It's recommended to use a larger buffer. For example, Go's net/http server uses a 64KB buffer for panic recovery.

Suggested change
buf := make([]byte, 1024)
buf := make([]byte, 64*1024)

n := runtime.Stack(buf, false)
err := fmt.Errorf("node panicked: %v\nstack trace:\n%s", r, buf[:n])
logger.Error().Interface("panic", r).Str("stacktrace", string(buf[:n])).Msg("Recovered from panic in node")
select {
case errCh <- err:
default:
Expand Down
18 changes: 13 additions & 5 deletions test/e2e/sut_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,25 +76,33 @@ func (s *SystemUnderTest) RunCmd(cmd string, args ...string) (string, error) {
}

// ExecCmd starts a process for the given command and manages it cleanup on test end.
func (s *SystemUnderTest) ExecCmd(cmd string, args ...string) {
func (s *SystemUnderTest) ExecCmd(cmd string, args ...string) *os.Process {
return s.ExecCmdWithLogPrefix("", cmd, args...)
}

// ExecCmdWithLogPrefix same as ExecCmd but prepends the given prefix to the log output
func (s *SystemUnderTest) ExecCmdWithLogPrefix(prefix, cmd string, args ...string) *os.Process {

executable := locateExecutable(cmd)
c := exec.Command( //nolint:gosec // used by tests only
executable,
args...,
)
c.Dir = WorkDir
s.watchLogs(c)
s.watchLogs(prefix, c)

err := c.Start()
require.NoError(s.t, err)
if s.debug {
s.logf("Exec cmd (pid: %d): %s %s", c.Process.Pid, executable, strings.Join(c.Args, " "))
s.logf("Exec cmd (pid: %d): %s %s", c.Process.Pid, executable, strings.Join(args, " "))
}
// cleanup when stopped
s.awaitProcessCleanup(c)
return c.Process
}

// AwaitNodeUp waits until a node is operational by checking both liveness and readiness.
// Fails tests when node is not up within the specified timeout.
func (s *SystemUnderTest) AwaitNodeUp(t *testing.T, rpcAddr string, timeout time.Duration) {
t.Helper()
t.Logf("Await node is up: %s", rpcAddr)
Expand Down Expand Up @@ -168,7 +176,7 @@ func (s *SystemUnderTest) awaitProcessCleanup(cmd *exec.Cmd) {
}()
}

func (s *SystemUnderTest) watchLogs(cmd *exec.Cmd) {
func (s *SystemUnderTest) watchLogs(prefix string, cmd *exec.Cmd) {
errReader, err := cmd.StderrPipe()
require.NoError(s.t, err)
outReader, err := cmd.StdoutPipe()
Expand All @@ -178,7 +186,7 @@ func (s *SystemUnderTest) watchLogs(cmd *exec.Cmd) {
logDir := filepath.Join(WorkDir, "testnet")
require.NoError(s.t, os.MkdirAll(logDir, 0o750))
testName := strings.ReplaceAll(s.t.Name(), "/", "-")
logfileName := filepath.Join(logDir, fmt.Sprintf("exec-%s-%s-%d.out", filepath.Base(cmd.Args[0]), testName, time.Now().UnixNano()))
logfileName := filepath.Join(logDir, prefix+fmt.Sprintf("exec-%s-%s-%d.out", filepath.Base(cmd.Args[0]), testName, time.Now().UnixNano()))
logfile, err := os.Create(logfileName)
require.NoError(s.t, err)
errReader = io.NopCloser(io.TeeReader(errReader, logfile))
Expand Down
Loading