Refactor osm-controller pod error handling and logging

Signed-off-by: Johnson Shi <[email protected]>

pkg/kubernetes/podhelper/container_logs.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"regexp"
 
+	"github.com/pkg/errors"
+
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 )
@@ -44,8 +46,7 @@ func HasNoBadLogs(client kubernetes.Interface, pod *v1.Pod, containerName string
 	}
 
 	if len(badLogLines) != 0 {
-		log.Error().Msgf("%s container of pod %s contains bad logs", containerName, pod.Name)
-		log.Error().Msg(badLogLines)
+		return errors.Errorf("%s container of pod %s contains bad log lines: %s", containerName, pod.Name, badLogLines)
 	}
 
 	return nil

pkg/osm/osm_controller_logs.go

@@ -4,9 +4,10 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/pkg/errors"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/openservicemesh/osm-health/pkg/common"
@@ -44,15 +45,21 @@ func (check NoBadOsmControllerLogsCheck) Run() error {
 	}
 	pods, err := check.client.CoreV1().Pods(check.osmControlPlaneNamespace).List(context.TODO(), listOptions)
 	if err != nil {
-		return fmt.Errorf("unable to list osm controller pods in namespace %s", check.osmControlPlaneNamespace)
+		return fmt.Errorf("unable to list %s pods in namespace %s", constants.OSMControllerName, check.osmControlPlaneNamespace)
 	}
 
+	osmControllerErrCount := 0
 	for i := range pods.Items {
 		if err := podhelper.HasNoBadLogs(check.client, &pods.Items[i], "osm-controller"); err != nil {
-			return err // TODO since we can have multiple osm-controller pods, should we return err on the first controller with bad logs?
+			osmControllerErrCount++
+			log.Error().Err(err)
 		}
 	}
 
+	if osmControllerErrCount != 0 {
+		return errors.Errorf("%s pods in namespace %s have %d errors", constants.OSMControllerName, check.osmControlPlaneNamespace, osmControllerErrCount)
+	}
+
 	return nil
 }