From ef13bd34050a1008101c931ab85b316c1c5dc8f5 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Fri, 30 Jun 2023 17:53:27 +0800 Subject: [PATCH] Add exit code log and possible memory shortage warning log for Restic command failure. Signed-off-by: Xun Jiang --- changelogs/unreleased/6459-blackpiglet | 1 + pkg/repository/restic/repository.go | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 changelogs/unreleased/6459-blackpiglet diff --git a/changelogs/unreleased/6459-blackpiglet b/changelogs/unreleased/6459-blackpiglet new file mode 100644 index 0000000000..26e0e38566 --- /dev/null +++ b/changelogs/unreleased/6459-blackpiglet @@ -0,0 +1 @@ +Add exit code log and possible memory shortage warning log for Restic command failure. \ No newline at end of file diff --git a/pkg/repository/restic/repository.go b/pkg/repository/restic/repository.go index 392caf2842..4f742d7164 100644 --- a/pkg/repository/restic/repository.go +++ b/pkg/repository/restic/repository.go @@ -18,6 +18,7 @@ package restic import ( "os" + "os/exec" "time" "github.com/pkg/errors" @@ -120,6 +121,17 @@ func (r *RepositoryService) exec(cmd *restic.Command, bsl *velerov1api.BackupSto "stderr": stderr, }).Debugf("Ran restic command") if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + r.log.Errorf("Restic command fail with ExitCode: %d. Process ID is %d, Exit error is: %s", exitError.ExitCode(), exitError.Pid(), exitError.String()) + // Golang's os.exec -1 ExitCode means signal kill. Usually this is caused + // by CGroup's OOM. Log a warning to notice user. + // https://github.com/golang/go/blob/master/src/os/exec_posix.go#L128-L136 + if exitError.ExitCode() == -1 { + r.log.Warnf("The ExitCode is -1, which means the process is terminated by signal. Usually this is caused by CGroup kill due to out of memory. Please check whether this is such information in the work nodes' dmesg log.") + } + } else { + r.log.Error("Error cannot be convert to ExitError format") + } return errors.Wrapf(err, "error running command=%s, stdout=%s, stderr=%s", cmd.String(), stdout, stderr) }