check(osm): Check for bad logs in osm controller pods

Signed-off-by: Johnson Shi <[email protected]>

pkg/connectivity/pod-to-pod.go

@@ -6,7 +6,7 @@ import (
 	"github.com/openservicemesh/osm-health/pkg/common"
 	"github.com/openservicemesh/osm-health/pkg/envoy"
 	"github.com/openservicemesh/osm-health/pkg/kubernetes/namespace"
-	"github.com/openservicemesh/osm-health/pkg/kubernetes/pod"
+	"github.com/openservicemesh/osm-health/pkg/kubernetes/podhelper"
 	"github.com/openservicemesh/osm-health/pkg/kuberneteshelper"
 	"github.com/openservicemesh/osm-health/pkg/osm"
 )
@@ -40,18 +40,18 @@ func PodToPod(fromPod *v1.Pod, toPod *v1.Pod) {
 		// Check source Pod's namespace
 		namespace.IsInjectEnabled(client, fromPod.Namespace),
 		namespace.IsMonitoredBy(client, fromPod.Namespace, meshName),
-		pod.HasMinExpectedContainers(fromPod, 3),
-		pod.HasExpectedEnvoyImage(fromPod),
-		pod.HasProxyUUIDLabel(fromPod),
-		pod.DoesNotHaveBadEvents(client, fromPod),
+		podhelper.HasMinExpectedContainers(fromPod, 3),
+		podhelper.HasExpectedEnvoyImage(fromPod),
+		podhelper.HasProxyUUIDLabel(fromPod),
+		podhelper.DoesNotHaveBadEvents(client, fromPod),
 
 		// Check destination Pod's namespace
 		namespace.IsInjectEnabled(client, toPod.Namespace),
 		namespace.IsMonitoredBy(client, toPod.Namespace, meshName),
-		pod.HasMinExpectedContainers(toPod, 3),
-		pod.HasExpectedEnvoyImage(toPod),
-		pod.HasProxyUUIDLabel(toPod),
-		pod.DoesNotHaveBadEvents(client, toPod),
+		podhelper.HasMinExpectedContainers(toPod, 3),
+		podhelper.HasExpectedEnvoyImage(toPod),
+		podhelper.HasProxyUUIDLabel(toPod),
+		podhelper.DoesNotHaveBadEvents(client, toPod),
 
 		// The source Envoy must have at least one endpoint for the destination Envoy.
 		envoy.HasDestinationEndpoints(srcConfigGetter),

pkg/envoy/envoy_container_logs.go

@@ -1,15 +1,13 @@
 package envoy
 
 import (
-	"bufio"
-	"context"
 	"fmt"
-	"regexp"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/openservicemesh/osm-health/pkg/common"
+	"github.com/openservicemesh/osm-health/pkg/kubernetes/podhelper"
 )
 
 // Verify interface compliance
@@ -21,7 +19,7 @@ type NoBadEnvoyLogsCheck struct {
 	pod    *v1.Pod
 }
 
-// HasNoBadEnvoyLogsCheck checks whether the envoy container of the pod has bad/error/warning log messages
+// HasNoBadEnvoyLogsCheck checks whether the envoy container of the pod has bad (fatal/error/warning/fail) log messages
 func HasNoBadEnvoyLogsCheck(client kubernetes.Interface, pod *v1.Pod) NoBadEnvoyLogsCheck {
 	return NoBadEnvoyLogsCheck{
 		client: client,
@@ -31,49 +29,12 @@ func HasNoBadEnvoyLogsCheck(client kubernetes.Interface, pod *v1.Pod) NoBadEnvoy
 
 // Info implements common.Runnable
 func (check NoBadEnvoyLogsCheck) Info() string {
-	return fmt.Sprintf("Checking whether pod %s has bad/error logs in envoy container", check.pod.Name)
+	return fmt.Sprintf("Checking whether pod %s has bad (fatal/error/warning/fail) logs in envoy container", check.pod.Name)
 }
 
 // Run implements common.Runnable
 func (check NoBadEnvoyLogsCheck) Run() error {
-	envoyLogsTailLines := int64(10)
-	envoyContainerName := "envoy"
-	podLogsOpt := v1.PodLogOptions{
-		Container: envoyContainerName,
-		Follow:    false,
-		Previous:  false,
-		TailLines: &envoyLogsTailLines,
-	}
-
-	request := check.client.CoreV1().Pods(check.pod.Namespace).GetLogs(check.pod.Name, &podLogsOpt)
-	envoyPodLogsReader, err := request.Stream(context.TODO())
-	if err != nil {
-		// If there are issues obtaining current container logs, return previously terminated container logs.
-		podLogsOpt.Previous = true
-		request := check.client.CoreV1().Pods(check.pod.Namespace).GetLogs(check.pod.Name, &podLogsOpt)
-		envoyPodLogsReader, err = request.Stream(context.TODO())
-		if err != nil {
-			return fmt.Errorf("could not obtain %s container logs of pod %s: %#v", envoyContainerName, check.pod.Name, err)
-		}
-	}
-	defer envoyPodLogsReader.Close() //nolint: errcheck,gosec
-
-	re := regexp.MustCompile("(?i)(error)|(warn)")
-	scanner := bufio.NewScanner(envoyPodLogsReader)
-	var badEnvoyLogLines string
-	for scanner.Scan() {
-		logLine := scanner.Text()
-		if re.MatchString(logLine) {
-			badEnvoyLogLines += logLine + "\n"
-		}
-	}
-
-	if len(badEnvoyLogLines) != 0 {
-		log.Error().Msgf("%s container of pod %s contains bad logs", envoyContainerName, check.pod.Name)
-		log.Error().Msg(badEnvoyLogLines)
-	}
-
-	return nil
+	return podhelper.HasNoBadLogs(check.client, check.pod, "envoy")
 }
 
 // Suggestion implements common.Runnable.

pkg/kubernetes/podhelper/container_logs.go

@@ -0,0 +1,52 @@
+package podhelper
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"regexp"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// HasNoBadLogs checks whether the logs of the pod container contain bad (fatal/error/warning/fail) logs
+func HasNoBadLogs(client kubernetes.Interface, pod *v1.Pod, containerName string) error {
+	logsTailLines := int64(10)
+	podLogsOpt := v1.PodLogOptions{
+		Container: containerName,
+		Follow:    false,
+		Previous:  false,
+		TailLines: &logsTailLines,
+	}
+
+	request := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogsOpt)
+	podLogsReader, err := request.Stream(context.TODO())
+	if err != nil {
+		// If there are issues obtaining current container logs, return previously terminated container logs.
+		podLogsOpt.Previous = true
+		request := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogsOpt)
+		podLogsReader, err = request.Stream(context.TODO())
+		if err != nil {
+			return fmt.Errorf("could not obtain %s container logs of pod %s: %#v", containerName, pod.Name, err)
+		}
+	}
+	defer podLogsReader.Close() //nolint: errcheck,gosec
+
+	re := regexp.MustCompile("(?i)(fatal)|(error)|(warn)|(fail)")
+	scanner := bufio.NewScanner(podLogsReader)
+	var badLogLines string
+	for scanner.Scan() {
+		logLine := scanner.Text()
+		if re.MatchString(logLine) {
+			badLogLines += logLine + "\n"
+		}
+	}
+
+	if len(badLogLines) != 0 {
+		log.Error().Msgf("%s container of pod %s contains bad logs", containerName, pod.Name)
+		log.Error().Msg(badLogLines)
+	}
+
+	return nil
+}

pkg/kubernetes/pod/containers.go → pkg/kubernetes/podhelper/containers.go

@@ -1,4 +1,4 @@
-package pod
+package podhelper
 
 import (
 	"fmt"

pkg/kubernetes/pod/containers_test.go → pkg/kubernetes/podhelper/containers_test.go

@@ -1,4 +1,4 @@
-package pod
+package podhelper
 
 import (
 	"testing"

pkg/kubernetes/pod/errors.go → pkg/kubernetes/podhelper/errors.go

@@ -1,4 +1,4 @@
-package pod
+package podhelper
 
 import (
 	"errors"

pkg/kubernetes/pod/events.go → pkg/kubernetes/podhelper/events.go

@@ -1,4 +1,4 @@
-package pod
+package podhelper
 
 import (
 	"context"

pkg/kubernetes/pod/labels.go → pkg/kubernetes/podhelper/labels.go

@@ -1,4 +1,4 @@
-package pod
+package podhelper
 
 import (
 	"fmt"

pkg/kubernetes/pod/labels_test.go → pkg/kubernetes/podhelper/labels_test.go

@@ -1,4 +1,4 @@
-package pod
+package podhelper
 
 import (
 	"testing"

pkg/kubernetes/podhelper/types.go

@@ -0,0 +1,7 @@
+package podhelper
+
+import (
+	"github.com/openservicemesh/osm-health/pkg/logger"
+)
+
+var log = logger.New("osm-health/kubernetes/pod")

pkg/osm/osm_controller_logs.go

@@ -0,0 +1,67 @@
+package osm
+
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/openservicemesh/osm-health/pkg/common"
+	"github.com/openservicemesh/osm-health/pkg/kubernetes/podhelper"
+	"github.com/openservicemesh/osm/pkg/constants"
+)
+
+// Verify interface compliance
+var _ common.Runnable = (*NoBadOsmControllerLogsCheck)(nil)
+
+// NoBadOsmControllerLogsCheck implements common.Runnable
+type NoBadOsmControllerLogsCheck struct {
+	client                   kubernetes.Interface
+	osmControlPlaneNamespace string
+}
+
+// HasNoBadOsmControllerLogsCheck checks whether the osm controller pods in the controller namespace have bad (fatal/error/warning/fail) log messages
+func HasNoBadOsmControllerLogsCheck(client kubernetes.Interface, osmControlPlaneNamespace string) NoBadOsmControllerLogsCheck {
+	return NoBadOsmControllerLogsCheck{
+		client:                   client,
+		osmControlPlaneNamespace: osmControlPlaneNamespace,
+	}
+}
+
+// Info implements common.Runnable
+func (check NoBadOsmControllerLogsCheck) Info() string {
+	return fmt.Sprintf("Checking whether namespace %s has bad (fatal/error/warning/fail) logs in osm controller pods", check.osmControlPlaneNamespace)
+}
+
+// Run implements common.Runnable
+func (check NoBadOsmControllerLogsCheck) Run() error {
+	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"app": constants.OSMControllerName}}
+	listOptions := metav1.ListOptions{
+		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
+	}
+	pods, err := check.client.CoreV1().Pods(check.osmControlPlaneNamespace).List(context.TODO(), listOptions)
+	if err != nil {
+		return fmt.Errorf("unable to list osm controller pods in namespace %s", check.osmControlPlaneNamespace)
+	}
+
+	for i := range pods.Items {
+		if err := podhelper.HasNoBadLogs(check.client, &pods.Items[i], "osm-controller"); err != nil {
+			return err // TODO since we can have multiple osm-controller pods, should we return err on the first controller with bad logs?
+		}
+	}
+
+	return nil
+}
+
+// Suggestion implements common.Runnable.
+func (check NoBadOsmControllerLogsCheck) Suggestion() string {
+	panic("implement me")
+}
+
+// FixIt implements common.Runnable.
+func (check NoBadOsmControllerLogsCheck) FixIt() error {
+	panic("implement me")
+}

pkg/osm/status.go

@@ -9,13 +9,12 @@ import (
 func ControlPlaneStatus(osmControlPlaneNamespace string) error {
 	log.Info().Msgf("Determining the status of the OSM control plane in namespace %s", osmControlPlaneNamespace)
 
-	_, err := kuberneteshelper.GetKubeClient()
+	client, err := kuberneteshelper.GetKubeClient()
 	if err != nil {
 		log.Err(err).Msg("Error creating Kubernetes client")
 	}
 
-	// TODO add checks like osm controller log checks
-	outcomes := common.Run()
+	outcomes := common.Run(HasNoBadOsmControllerLogsCheck(client, osmControlPlaneNamespace))
 
 	common.Print(outcomes...)